diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..70c57bd --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# export $(grep -v '^#' .env | xargs) + +SOURCE_HOST= +SOURCE_ORG= +GH_SOURCE_PAT= + +TARGET_HOST= +TARGET_ORG= +GH_TARGET_PAT= diff --git a/.gitignore b/.gitignore index d52ace2..a465620 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ node_modules* test*.js test*.sh +temp +tool +.env diff --git a/scripts/migrate-npm-packages-between-github-instances.sh b/scripts/migrate-npm-packages-between-github-instances.sh index 4fbcdd5..f774acf 100755 --- a/scripts/migrate-npm-packages-between-github-instances.sh +++ b/scripts/migrate-npm-packages-between-github-instances.sh @@ -11,27 +11,27 @@ # Example: ./migrate-npm-packages-between-github-instances.sh joshjohanning-org github.com joshjohanning-emu github.com | tee output.log # # Notes: -# - Mapping the npm package to a repo is optional. +# - Mapping the npm package to a repo is optional. # - If there is a repo that exists in the target with the same repo name, it will map it # - If the repo doesn't exist, the package will still import but won't be mapped to a repo # - See ./failed-packages.txt for any packages that failed to import set -e -if [ $# -ne "4" ]; then - echo "Usage: $0 " - exit 1 +if [ $# -lt "4" ]; then + echo "Usage: $0 " + exit 1 fi # make sure env variables are defined if [ -z "$GH_SOURCE_PAT" ]; then - echo "Error: set GH_SOURCE_PAT env var" - exit 1 + echo "Error: set GH_SOURCE_PAT env var" + exit 1 fi if [ -z "$GH_TARGET_PAT" ]; then - echo "Error: set GH_TARGET_PAT env var" - exit 1 + echo "Error: set GH_TARGET_PAT env var" + exit 1 fi echo "..." @@ -40,6 +40,11 @@ SOURCE_ORG=$1 SOURCE_HOST=$2 TARGET_ORG=$3 TARGET_HOST=$4 +# Optional: Set CUTOFF_DATE to migrate only package versions created on or after this date. +# Format: YYYY-MM-DDTHH:MM:SSZ (e.g., 2023-03-13T00:00:00Z) +# Example: To migrate versions from the past year, use: $(date -u -v-1y +"%Y-%m-%dT%H:%M:%SZ") +# If CUTOFF_DATE is unset or empty, all available versions will be migrated. +CUTOFF_DATE=$5 # create temp dir mkdir -p ./temp @@ -47,7 +52,7 @@ cd ./temp temp_dir=$(pwd) # set up .npmrc for target org -echo @$TARGET_ORG:registry=https://npm.pkg.$TARGET_HOST/ > $temp_dir/.npmrc && echo "//npm.pkg.$TARGET_HOST/:_authToken=$GH_TARGET_PAT" >> $temp_dir/.npmrc +echo @$TARGET_ORG:registry=https://npm.pkg.$TARGET_HOST/ >$temp_dir/.npmrc && echo "//npm.pkg.$TARGET_HOST/:_authToken=$GH_TARGET_PAT" >>$temp_dir/.npmrc packages=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages?package_type=npm" -q '.[] | .name + " " + .repository.name') @@ -57,32 +62,44 @@ echo "$packages" | while IFS= read -r response; do repo_name=$(echo "$response" | cut -d ' ' -f 2) echo "org: $SOURCE_ORG repo: $repo_name --> package name $package_name" - - versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" -q '.[] | .name' | sort -V) - for version in $versions - do + + # Cache package metadata to avoid multiple API calls for each version + curl -H "Authorization: token $GH_SOURCE_PAT" -Ls "https://npm.pkg.github.com/@$SOURCE_ORG/$package_name" >"${temp_dir}/${package_name}.json" + + # Fetch versions, filter by date only if CUTOFF_DATE is set + if [ -n "$CUTOFF_DATE" ]; then + versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" | + jq -r --arg cutoff "$CUTOFF_DATE" '.[] | select(.created_at >= $cutoff) | .name' | + sort -V) + else + versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" | + jq -r '.[].name' | + sort -V) + fi + + for version in $versions; do echo "$version" # get url of tarball - url=$(curl -H "Authorization: token $GH_SOURCE_PAT" -Ls https://npm.pkg.github.com/@$SOURCE_ORG/$package_name | jq --arg version $version -r '.versions[$version].dist.tarball') + url=$(jq --arg version $version -r '.versions[$version].dist.tarball' "${temp_dir}/${package_name}.json") # check for error if [ "$url" == "null" ]; then - echo "ERROR: version $version not found for package $package_name" - echo "NOTE: Make sure you have the proper scopes for gh; ie run this: gh auth refresh -h github.com -s read:packages" - exit 1 + echo "ERROR: version $version not found for package $package_name" + echo "NOTE: Make sure you have the proper scopes for gh; ie run this: gh auth refresh -h github.com -s read:packages" + continue fi - # download + # download curl -sS -H "Authorization: token $GH_SOURCE_PAT" -L -o $package_name-$version.tgz $url - + # untar mkdir -p ./$package_name-$version # if you run into permissions issue, add a `sudo` here tar xzf $package_name-$version.tgz -C $package_name-$version cd $package_name-$version/package perl -pi -e "s/$SOURCE_ORG/$TARGET_ORG/ig" package.json - npm publish --ignore-scripts --userconfig $temp_dir/.npmrc || echo "skipped package due to failure: $package_name-$version.tgz" >> ./failed-packages.txt + npm publish --ignore-scripts --userconfig $temp_dir/.npmrc || echo "skipped package due to failure: $package_name-$version.tgz" >>./failed-packages.txt cd ./../../ done