diff --git a/TEMPLATE_hermes_github_to_zenodo.yml b/TEMPLATE_hermes_github_to_zenodo.yml index 3f6b8ab..cfa68f9 100644 --- a/TEMPLATE_hermes_github_to_zenodo.yml +++ b/TEMPLATE_hermes_github_to_zenodo.yml @@ -48,7 +48,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install hermes + - run: pip install git+https://github.com/softwarepub/hermes.git - run: hermes harvest - run: hermes process - run: hermes curate @@ -92,12 +92,12 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install hermes + - run: pip install git+https://github.com/softwarepub/hermes.git # ADAPT # If you want to publish artifacts (e.g., a zipped snapshot of your repository), # you can prepare this here. - - run: git archive --format zip HEAD src > showcase.zip + - run: git archive --format zip HEAD src > artifact.zip # Run the HERMES deposition and postprocessing steps. # ADAPT @@ -106,7 +106,9 @@ jobs: # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), # rather than a new version of the same collection! - - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file showcase.zip --file README.md + - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file artifact.zip --file README.md + - if: ${{ env.ZENODO_TOKEN_REFRESH != '' }} + run: hermes init --only-set-refresh-token --github-token ${{ secrets.GITHUB_TOKEN }} # ADAPT # Remove this command if you don't want to do any postprocessing diff --git a/gitlab/hermes-ci.yml b/gitlab/hermes-ci.yml index 9bc761e..7bac361 100644 --- a/gitlab/hermes-ci.yml +++ b/gitlab/hermes-ci.yml @@ -74,7 +74,7 @@ python -m venv .hermes-env fi - . .hermes-env/bin/activate - - pip install hermes + - pip install git+https://github.com/softwarepub/hermes.git@feature/init-command tags: - docker artifacts: diff --git a/init/TEMPLATE_hermes_github_to_zenodo.yml b/init/TEMPLATE_hermes_github_to_zenodo.yml new file mode 100644 index 0000000..dcd2211 --- /dev/null +++ b/init/TEMPLATE_hermes_github_to_zenodo.yml @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf +# +# SPDX-License-Identifier: CC0-1.0 + +############################################################################################################### +# TEMPLATE! To use this template, do the following: +# +# 1. Copy this file into the .github/workflows/ directory in your repository and remove the 'TEMPLATE_' prefix. +# 2. Go through the file and carefully adapt to your needs, as marked by an '# ADAPT' comment. +############################################################################################################### + +name: Software Publication + +on: + # ADAPT + # See the events you can react to at https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows + push: + branches: + - {%push_branch%} + # NOTE: Do not delete the trigger on closed pull requests, the HERMES workflow needs this. + pull_request: + types: + - closed + +jobs: + hermes-prepare: + name: Prepare Metadata for Curation + runs-on: ubuntu-latest + # This condition becomes much easier when we only react to push events on the release branch. + # We still need to exclude the merge commit push of the post processing PR + + # ADAPT + # Depending on the event you react to in the 'on:' section above, you will need to adapt this + # to react on the specific events. + # NOTE: You will probably still need to keep the exclusion check for commit messages provided by the workflow ('hermes/'/'hermes/post'). + if: > + github.event_name == 'push' && ! ( + startsWith(github.ref_name, 'hermes/') || + contains(github.event.head_commit.message, 'hermes/post') + ) + + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: pip install hermes +{%pip_install_plugins_github%} + - run: hermes harvest + - run: hermes process + - run: hermes curate + + - run: | + # Cache current branch for PR close job + git branch --show-current > .hermes/curate/target_branch + + # Shorten the SHA for the PR title + SHORT_SHA=$(echo "$GITHUB_SHA" | cut -c -8) + echo "SHORT_SHA=$SHORT_SHA" >> "$GITHUB_ENV" + + # Create a curation branch + git branch -c "hermes/curate-$SHORT_SHA" + git push origin "hermes/curate-$SHORT_SHA" + + # Explicitly add to-be-curated metadata (which is ignored via .gitignore!) + git add -f .hermes/curate + - uses: peter-evans/create-pull-request@v5 + with: + base: hermes/curate-${{ env.SHORT_SHA }} + branch: hermes/curate-result-${{ env.SHORT_SHA }} + title: Metadata Curation for Commit ${{ env.SHORT_SHA }} + body: | + Please carefully review the attached metadata. + If you are satisfied with the result, you may merge this PR, which will trigger publication. + (Any temporary branches will be cleaned up.) + delete-branch: true + + hermes-curate: + name: Publish Software with Curated Metadata + if: github.event.pull_request.merged == true && startsWith( github.base_ref , 'hermes/curate-') + + runs-on: ubuntu-latest + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: pip install hermes +{%pip_install_plugins_github%} + + # ADAPT + # If you want to publish artifacts (e.g., a zipped snapshot of your repository), + # you can prepare this here. + - run: git archive --format zip HEAD {%deposit_zip_files%} > {%deposit_zip_name%} + + # Run the HERMES deposition and postprocessing steps. + # ADAPT + # 1. You need to have an authentication token for your target publication platform + # as a GitHub secret in your repository (in this example, this is called ZENODO_SANDBOX). + # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. + # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), + # rather than a new version of the same collection! + - run: hermes deposit {%deposit_initial%} -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file {%deposit_zip_name%} {%deposit_extra_files%} + + # ADAPT + # Remove this command if you don't want to do any postprocessing + - run: hermes postprocess + + # ADAPT + # If you don't want to run postprocessing, remove this complete section (next '-run' and 'uses: peter-evans/...' bullets). + # + # Note 1: We change the base branch here for the PR. This flow runs so far within the "curated-metadata-*" branch, + # but now we want to add the changes done by deposit and post processing to the branch that was initially + # meant to be published using HERMES. + # Note 2: The create-pull-request action will NOT inherit the commits we did in the previous job. It will only look at the + # changes within this local workspace we did *now*. + - run: echo "TARGET_BRANCH=$(cat .hermes/curate/target_branch)" >> "$GITHUB_ENV" + - uses: peter-evans/create-pull-request@v5 + with: + branch: hermes/post-${{ github.run_id }} + base: ${{ env.TARGET_BRANCH }} + title: Review hermes post-processing results + body: | + This is an automated pull request created by HERMES post-processing. + + Please carefully review the changes and finally merge them into your + + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done + # TODO: if: failure() --- delete the curation branches when the deposition failed + + + hermes-cleanup: + name: Cleanup aborted curation branches + if: github.event.pull_request.merged == false && startsWith( github.base_ref , 'hermes/curate-') + + runs-on: ubuntu-latest + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done diff --git a/init/TEMPLATE_hermes_gitlab_to_zenodo.yml b/init/TEMPLATE_hermes_gitlab_to_zenodo.yml new file mode 100644 index 0000000..13a6572 --- /dev/null +++ b/init/TEMPLATE_hermes_gitlab_to_zenodo.yml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf +# +# SPDX-License-Identifier: CC0-1.0 + +include: gitlab/hermes-ci.yml + +# "Call-back" to create archive +# ADAPT with commands that create a file to upload. The filename is provided as $HERMES_ARCHIVE_NAME. +.hermes_create_deposition: &hermes_create_deposition + - git fetch origin "$MR_TARGET_BRANCH" + - git archive --format zip "origin/$MR_TARGET_BRANCH" > $HERMES_ARCHIVE_NAME + +hermes_curate: + extends: + - .hermes_curate + +hermes_deposit: + extends: + - .hermes_deposit diff --git a/init/hermes-ci.yml b/init/hermes-ci.yml new file mode 100644 index 0000000..7945727 --- /dev/null +++ b/init/hermes-ci.yml @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf +# +# SPDX-License-Identifier: CC0-1.0 + +# This template provides a set of template workflows that can be used to easily integrate +# the hermes workflow into your GitLab CI. +# +# The following templates are available: +# +# - ˋ.hermes_curateˋ: Prepare metadata for curation in a merge request. +# - ˋ.hermes_depositˋ: Deposit the curated metadata and run post-processing. +# - ˋ.hermes_cleanupˋ: Remove transient branches created by the workflow. +# +# The following variables can be used to tune the workflows: +# +# - ˋMR_BASE_REFˋ: The git reference to start hermes workflow from. +# - ˋMR_POST_BRANCHˋ: The branch to merge post-processing results into. +# This defaults to $CI_COMMIT_BRANCH if set, else $CI_DEFAULT_BRANCH. +# Needs to be defined for ˋ.hermes_curateˋ. +# - ˋMR_TITLEˋ: Title for the merge request. +# - ˋMR_DESCRIPTIONˋ: Description for the merge request. +# - ˋMR_COMMIT_MESSAGEˋ: Commit message for the autoated commit (if any). + +# "Call-back" to create archive +# ADAPT with commands that create a file to upload. The filename is provided as $HERMES_ARCHIVE_NAME. +.hermes_create_deposition: &hermes_create_deposition + - git fetch origin "$MR_TARGET_BRANCH" + - git archive --format zip "origin/$MR_TARGET_BRANCH" {%deposit_zip_files%} > $HERMES_ARCHIVE + + +# Basic building blocks of hermes jobs +# - Set up git with a valid author and configure URL to allow pushes. +.hermes_prepare_git: &hermes_prepare_git + - git config user.name "$GITLAB_USER_NAME" + - git config user.email "$GITLAB_USER_EMAIL" + - git remote set-url origin "${CI_SERVER_PROTOCOL}://__token__:${HERMES_PUSH_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/${CI_PROJECT_PATH}.git" + +# - Switch to new MR_SOURCE_BRANCH. +.hermes_switch_branch: &hermes_switch_branch + - git branch "$MR_SOURCE_BRANCH" "origin/$MR_TARGET_BRANCH" + - git switch "$MR_SOURCE_BRANCH" + +# - Commit changes and create a matching merge request. +.hermes_create_merge_request: &hermes_create_merge_request + - git commit -m "$MR_COMMIT_MESSAGE" + - >- + git push origin "$MR_SOURCE_BRANCH" + -o merge_request.create + -o merge_request.target="$MR_TARGET_BRANCH" + -o merge_request.title="$MR_TITLE" + -o merge_request.description="$MR_DESCRIPTION" + -o merge_request.remove_source_branch + +# - Delete all branches created by hermes for curation. +.hermes_cleanup_branches: &hermes_cleanup_branches + - | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done + +# Base job template for hermes +.hermes_job: + stage: deploy + image: python:3.10 + cache: + key: hermes + paths: + - .hermes-env + before_script: + - *hermes_prepare_git + - | + if test ! -d .hermes-env + then + python -m venv .hermes-env + fi + - . .hermes-env/bin/activate + - pip install hermes +{%pip_install_plugins_gitlab%} + tags: + - docker + artifacts: + paths: [".hermes/"] + +# hermes metadata preparation - this job creates a MR that will trigger deposition when merged. +.hermes_curate: + extends: + - .hermes_job + variables: + MR_BASE_REF: "origin/$CI_COMMIT_BRANCH" + MR_SOURCE_BRANCH: hermes/curate-result-$CI_COMMIT_SHORT_SHA + MR_TARGET_BRANCH: hermes/curate-$CI_COMMIT_SHORT_SHA + MR_COMMIT_MESSAGE: "[hermes] Add metadata for curation" + MR_TITLE: Metadata Curation for Commit $CI_COMMIT_SHORT_SHA + MR_DESCRIPTION: >- + Please carefully review the attached metadata. + If you are satisfied with the result, you may merge this MR, which will trigger publication. + (Any temporary branches will be cleaned up.) + rules: + - exists: + - .hermes/curate/target_branch + when: never + - if: $CI_COMMIT_TITLE =~ /^Merge branch 'hermes\/post-[0-9a-f]{8}' into/ + when: never + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: + # Run hermes pipeline + - hermes harvest + - hermes process + - hermes curate + + # Store BASE ref for post-processing + - | + if [ -z "$MR_POST_BRANCH" ] + then + if [ ! -z "$CI_COMMIT_BRANCH" ] + then + MR_POST_BRANCH=$CI_COMMIT_BRANCH + else + MR_POST_BRANCH=$CI_DEFAULT_BRANCH + fi + fi + echo $MR_POST_BRANCH > .hermes/curate/target_branch + + # Create target branch for curation merge + - git branch "$MR_TARGET_BRANCH" "$MR_BASE_REF" + - git push origin "$MR_TARGET_BRANCH" + + # Create curation branch and merge request + - *hermes_switch_branch + - git add -f .hermes/curate + - *hermes_create_merge_request + +# hermes deposition and post-processing - this job creates a MR to re-integrate the results of post-processing. +.hermes_deposit: + stage: deploy + extends: + - .hermes_job + variables: + HERMES_ARCHIVE: {%deposit_zip_name%} + MR_SOURCE_BRANCH: hermes/post-$CI_COMMIT_SHORT_SHA + MR_COMMIT_MESSAGE: "[hermes] Add post-processing results" + MR_TITLE: Review hermes post-processing results + MR_DESCRIPTION: >- + This is an automated pull request created by HERMES post-processing. + Please carefully review the changes and finally merge them into your + rules: + - if: $CI_PIPELINE_SOURCE != "push" || $CI_COMMIT_BRANCH !~ /^hermes\/curate-[0-9a-f]{8}/ + when: never + - exists: + - .hermes/curate/target_branch + script: + # Restore target branch + - MR_TARGET_BRANCH="$(cat .hermes/curate/target_branch)" + + # Invoke callback to create $HERMES_ARCHIVE_NAME and publish + - *hermes_create_deposition + - hermes deposit {%deposit_initial%} -O invenio_rdm.auth_token "$ZENODO_TOKEN" --file "$HERMES_ARCHIVE" {%deposit_extra_files%} + + # Prepare clean branch for post-processing and run it + - *hermes_switch_branch + - hermes postprocess + + # Put all changes into a merge request (all but .hermes-env) + - git add . + - git reset .hermes-env + - *hermes_create_merge_request + after_script: + - *hermes_cleanup_branches + +.hermes_cleanup: + script: + - *hermes_cleanup_branches