From 2a98a62ce1f6d1d1b524d8f7d381db08510e5a48 Mon Sep 17 00:00:00 2001 From: Ahmed Magdy Date: Fri, 2 May 2025 09:54:10 +0200 Subject: [PATCH 1/4] Update .gitignore to ignore temp & tool folders --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d52ace2..665fe77 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ node_modules* test*.js test*.sh +temp +tool From 69bef6f7119adbc321b28ba60602317a8bab0f61 Mon Sep 17 00:00:00 2001 From: Ahmed Magdy Date: Fri, 2 May 2025 09:55:02 +0200 Subject: [PATCH 2/4] Add a .env.example file with some variables that are used in scripts --- .env.example | 9 +++++++++ .gitignore | 1 + 2 files changed, 10 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..70c57bd --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# export $(grep -v '^#' .env | xargs) + +SOURCE_HOST= +SOURCE_ORG= +GH_SOURCE_PAT= + +TARGET_HOST= +TARGET_ORG= +GH_TARGET_PAT= diff --git a/.gitignore b/.gitignore index 665fe77..a465620 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ test*.js test*.sh temp tool +.env From ba58ba9a82b0d7424c966f262d6e63b741209d7c Mon Sep 17 00:00:00 2001 From: Ahmed Magdy Date: Fri, 2 May 2025 13:08:56 +0200 Subject: [PATCH 3/4] Update migrate-npm-packages-between-github-instances.sh to have a cutoff date --- ...grate-npm-packages-between-github-instances.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/migrate-npm-packages-between-github-instances.sh b/scripts/migrate-npm-packages-between-github-instances.sh index 4fbcdd5..d60fd82 100755 --- a/scripts/migrate-npm-packages-between-github-instances.sh +++ b/scripts/migrate-npm-packages-between-github-instances.sh @@ -18,7 +18,7 @@ set -e -if [ $# -ne "4" ]; then +if [ $# -lt "4" ]; then echo "Usage: $0 " exit 1 fi @@ -40,6 +40,7 @@ SOURCE_ORG=$1 SOURCE_HOST=$2 TARGET_ORG=$3 TARGET_HOST=$4 +CUTOFF_DATE=${5:-$(date -u -v-1y +"%Y-%m-%dT%H:%M:%SZ")} # format YYYY-MM-DDTHH:MM:SSZ ex: 2023-03-13T00:00:00Z # create temp dir mkdir -p ./temp @@ -57,20 +58,24 @@ echo "$packages" | while IFS= read -r response; do repo_name=$(echo "$response" | cut -d ' ' -f 2) echo "org: $SOURCE_ORG repo: $repo_name --> package name $package_name" - - versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" -q '.[] | .name' | sort -V) + + # Cache package metadata to avoid multiple API calls for each version + curl -H "Authorization: token $GH_SOURCE_PAT" -Ls "https://npm.pkg.github.com/@$SOURCE_ORG/$package_name" > "${temp_dir}/${package_name}.json" + + versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" | jq -r --arg cutoff "$CUTOFF_DATE" '.[] | select(.created_at >= $cutoff) | .name' | sort -V) + for version in $versions do echo "$version" # get url of tarball - url=$(curl -H "Authorization: token $GH_SOURCE_PAT" -Ls https://npm.pkg.github.com/@$SOURCE_ORG/$package_name | jq --arg version $version -r '.versions[$version].dist.tarball') + url=$(jq --arg version $version -r '.versions[$version].dist.tarball' "${temp_dir}/${package_name}.json") # check for error if [ "$url" == "null" ]; then echo "ERROR: version $version not found for package $package_name" echo "NOTE: Make sure you have the proper scopes for gh; ie run this: gh auth refresh -h github.com -s read:packages" - exit 1 + continue; fi # download From 2521168757dee13c03bcd02026aaf9dffe52c04b Mon Sep 17 00:00:00 2001 From: Ahmed Magdy Date: Tue, 17 Jun 2025 12:56:10 +0200 Subject: [PATCH 4/4] Update migrate-npm-packages-between-github-instances.sh allowing CUTOFF_DATE default to be empty, which in turns will migrate all versions (keeping existing behaviour) --- ...e-npm-packages-between-github-instances.sh | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/scripts/migrate-npm-packages-between-github-instances.sh b/scripts/migrate-npm-packages-between-github-instances.sh index d60fd82..f774acf 100755 --- a/scripts/migrate-npm-packages-between-github-instances.sh +++ b/scripts/migrate-npm-packages-between-github-instances.sh @@ -11,7 +11,7 @@ # Example: ./migrate-npm-packages-between-github-instances.sh joshjohanning-org github.com joshjohanning-emu github.com | tee output.log # # Notes: -# - Mapping the npm package to a repo is optional. +# - Mapping the npm package to a repo is optional. # - If there is a repo that exists in the target with the same repo name, it will map it # - If the repo doesn't exist, the package will still import but won't be mapped to a repo # - See ./failed-packages.txt for any packages that failed to import @@ -19,19 +19,19 @@ set -e if [ $# -lt "4" ]; then - echo "Usage: $0 " - exit 1 + echo "Usage: $0 " + exit 1 fi # make sure env variables are defined if [ -z "$GH_SOURCE_PAT" ]; then - echo "Error: set GH_SOURCE_PAT env var" - exit 1 + echo "Error: set GH_SOURCE_PAT env var" + exit 1 fi if [ -z "$GH_TARGET_PAT" ]; then - echo "Error: set GH_TARGET_PAT env var" - exit 1 + echo "Error: set GH_TARGET_PAT env var" + exit 1 fi echo "..." @@ -40,7 +40,11 @@ SOURCE_ORG=$1 SOURCE_HOST=$2 TARGET_ORG=$3 TARGET_HOST=$4 -CUTOFF_DATE=${5:-$(date -u -v-1y +"%Y-%m-%dT%H:%M:%SZ")} # format YYYY-MM-DDTHH:MM:SSZ ex: 2023-03-13T00:00:00Z +# Optional: Set CUTOFF_DATE to migrate only package versions created on or after this date. +# Format: YYYY-MM-DDTHH:MM:SSZ (e.g., 2023-03-13T00:00:00Z) +# Example: To migrate versions from the past year, use: $(date -u -v-1y +"%Y-%m-%dT%H:%M:%SZ") +# If CUTOFF_DATE is unset or empty, all available versions will be migrated. +CUTOFF_DATE=$5 # create temp dir mkdir -p ./temp @@ -48,7 +52,7 @@ cd ./temp temp_dir=$(pwd) # set up .npmrc for target org -echo @$TARGET_ORG:registry=https://npm.pkg.$TARGET_HOST/ > $temp_dir/.npmrc && echo "//npm.pkg.$TARGET_HOST/:_authToken=$GH_TARGET_PAT" >> $temp_dir/.npmrc +echo @$TARGET_ORG:registry=https://npm.pkg.$TARGET_HOST/ >$temp_dir/.npmrc && echo "//npm.pkg.$TARGET_HOST/:_authToken=$GH_TARGET_PAT" >>$temp_dir/.npmrc packages=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages?package_type=npm" -q '.[] | .name + " " + .repository.name') @@ -60,12 +64,20 @@ echo "$packages" | while IFS= read -r response; do echo "org: $SOURCE_ORG repo: $repo_name --> package name $package_name" # Cache package metadata to avoid multiple API calls for each version - curl -H "Authorization: token $GH_SOURCE_PAT" -Ls "https://npm.pkg.github.com/@$SOURCE_ORG/$package_name" > "${temp_dir}/${package_name}.json" - - versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" | jq -r --arg cutoff "$CUTOFF_DATE" '.[] | select(.created_at >= $cutoff) | .name' | sort -V) - - for version in $versions - do + curl -H "Authorization: token $GH_SOURCE_PAT" -Ls "https://npm.pkg.github.com/@$SOURCE_ORG/$package_name" >"${temp_dir}/${package_name}.json" + + # Fetch versions, filter by date only if CUTOFF_DATE is set + if [ -n "$CUTOFF_DATE" ]; then + versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" | + jq -r --arg cutoff "$CUTOFF_DATE" '.[] | select(.created_at >= $cutoff) | .name' | + sort -V) + else + versions=$(GH_HOST="$SOURCE_HOST" GH_TOKEN=$GH_SOURCE_PAT gh api --paginate "/orgs/$SOURCE_ORG/packages/npm/$package_name/versions" | + jq -r '.[].name' | + sort -V) + fi + + for version in $versions; do echo "$version" # get url of tarball @@ -73,21 +85,21 @@ echo "$packages" | while IFS= read -r response; do # check for error if [ "$url" == "null" ]; then - echo "ERROR: version $version not found for package $package_name" - echo "NOTE: Make sure you have the proper scopes for gh; ie run this: gh auth refresh -h github.com -s read:packages" - continue; + echo "ERROR: version $version not found for package $package_name" + echo "NOTE: Make sure you have the proper scopes for gh; ie run this: gh auth refresh -h github.com -s read:packages" + continue fi - # download + # download curl -sS -H "Authorization: token $GH_SOURCE_PAT" -L -o $package_name-$version.tgz $url - + # untar mkdir -p ./$package_name-$version # if you run into permissions issue, add a `sudo` here tar xzf $package_name-$version.tgz -C $package_name-$version cd $package_name-$version/package perl -pi -e "s/$SOURCE_ORG/$TARGET_ORG/ig" package.json - npm publish --ignore-scripts --userconfig $temp_dir/.npmrc || echo "skipped package due to failure: $package_name-$version.tgz" >> ./failed-packages.txt + npm publish --ignore-scripts --userconfig $temp_dir/.npmrc || echo "skipped package due to failure: $package_name-$version.tgz" >>./failed-packages.txt cd ./../../ done