From 1cf8bb658ad9a8a207685c04c5b0fa5baf340fba Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Tue, 5 Mar 2024 11:08:38 +0800 Subject: [PATCH 01/18] add migration code to download from one instance and upload to another --- cmd/integrationtest/command.go | 4 ++- pkg/buildtest/run.go | 61 ++++++++++++++++++++++++++++++++-- pkg/integrationtest/run.go | 4 +-- pkg/promotetest/run.go | 4 +++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/cmd/integrationtest/command.go b/cmd/integrationtest/command.go index 2e3fa13..a719a42 100644 --- a/cmd/integrationtest/command.go +++ b/cmd/integrationtest/command.go @@ -40,11 +40,12 @@ func NewIntegrationTestCmd() *cobra.Command { keepPod, _ := cmd.Flags().GetBool("keepPod") sidecar, _ := cmd.Flags().GetBool("sidecar") indyProxyUrl, _ := cmd.Flags().GetString("indyProxyUrl") + migrateTargetIndy, _ := cmd.Flags().GetString("migrateTargetIndy") metaCheckRepo := "" if len(args) >= 5 { metaCheckRepo = args[4] } - integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl) + integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl, migrateTargetIndy) }, } @@ -53,6 +54,7 @@ func NewIntegrationTestCmd() *cobra.Command { exec.Flags().BoolP("keepPod", "k", false, "Keep the pod after test to debug.") exec.Flags().BoolP("sidecar", "s", false, "Send requests through sidecar.") exec.Flags().StringP("indyProxyUrl", "p", "", "Indy generic proxy url.") + exec.Flags().StringP("migrateTargetIndy", "m", "", "Migrate download artifacts to new Indy url") return exec } diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 27ba7b1..ee01209 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -7,6 +7,8 @@ import ( "path" "strings" + "github.com/commonjava/indy-tests/pkg/promotetest" + common "github.com/commonjava/indy-tests/pkg/common" ) @@ -14,6 +16,7 @@ const ( TMP_DOWNLOAD_DIR = "/tmp/download" TMP_UPLOAD_DIR = "/tmp/upload" PROXY_ = "proxy-" + //migrateTargetIndyHost = "indy-gateway-indy--test.apps.gpc.ocp-hub.prod.psi.redhat.com" ) func Run(originalIndy, foloId, replacement, targetIndy, packageType string, processNum int) { @@ -23,11 +26,11 @@ func Run(originalIndy, foloId, replacement, targetIndy, packageType string, proc } foloTrackContent := common.GetFoloRecord(origIndy, foloId) newBuildName := common.GenerateRandomBuildName() - DoRun(originalIndy, targetIndy, "", packageType, newBuildName, foloTrackContent, nil, processNum, false, false) + DoRun(originalIndy, targetIndy, "", "", packageType, newBuildName, foloTrackContent, nil, processNum, false, false) } // Create the repo structure and do the download/upload -func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName string, foloTrackContent common.TrackedContent, +func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageType, newBuildName string, foloTrackContent common.TrackedContent, additionalRepos []string, processNum int, clearCache, dryRun bool) bool { @@ -40,6 +43,13 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName str os.Exit(1) } + migrateEnabled := (migrateTargetIndy != "") + if migrateEnabled { + migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) + fmt.Printf("Migrate to host %s", migrateTargetIndyHost) + prepareIndyRepos("http://"+migrateTargetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) + } + trackingId := foloTrackContent.TrackingKey.Id downloadDir, uploadDir := prepareDownUploadDirectories(trackingId, clearCache) @@ -64,12 +74,57 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName str } return success } + + migrateFunc := func(md5str, originalArtiURL, targetArtiURL string, migrateTargetArtiURL string) bool { + fileLoc := path.Join(downloadDir, path.Base(targetArtiURL)) + if dryRun { + fmt.Printf("Dry run download, url: %s\n", targetArtiURL) + return true + } + success := false + success, _ = common.DownloadFile(targetArtiURL, fileLoc) + if success { + common.Md5Check(fileLoc, md5str) + if dryRun { + fmt.Printf("Dry run upload, url: %s\n", migrateTargetArtiURL) + return true + } + common.UploadFile(migrateTargetArtiURL, fileLoc) + } + return success + } + broken := false if len(downloads) > 0 { fmt.Println("Start handling downloads artifacts.") fmt.Printf("==========================================\n\n") - if processNum > 1 { + if processNum > 1 && !migrateEnabled { broken = !common.ConcurrentRun(processNum, downloads, downloadFunc) + } else if migrateEnabled { + migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) + migrateUploads := prepareDownloadEntriesByFolo(migrateTargetIndyHost, newBuildName, packageType, foloTrackContent, additionalRepos, proxyEnabled) + paths := []string{} + rhpaths := []string{} + for i, down := range downloads { + fmt.Print(i) + broken = !migrateFunc(down[0], down[1], down[2], migrateUploads[i][2]) + if broken { + break + } else { + if strings.Contains(i, "redhat"){ + rhpaths = append(rhpaths, i) + } else { + paths = append(paths, i) + } + } + } + targetStore := packageType+":hosted:shared-imports" + rhTargetStore := packageType+":hosted:pnc-builds" + sourceStore := packageType+":hosted:"+newBuildName + promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, targetStore, paths, false) + if rhpaths != nil{ + promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, rhTargetStore, rhpaths, false) + } } else { for _, down := range downloads { broken = !downloadFunc(down[0], down[1], down[2]) diff --git a/pkg/integrationtest/run.go b/pkg/integrationtest/run.go index 97230a7..45b5543 100644 --- a/pkg/integrationtest/run.go +++ b/pkg/integrationtest/run.go @@ -56,7 +56,7 @@ const ( * j. Retrieve the metadata files from step #f again, check that the new version is gone * k. Clean up. Delete the build group G and the hosted repo A. Delete folo record. */ -func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo string, clearCache, dryRun, keepPod, sidecar bool, indyProxyUrl string) { +func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo string, clearCache, dryRun, keepPod, sidecar bool, indyProxyUrl string, migrateTargetIndy string) { if indyProxyUrl != "" { fmt.Println("Enable generic proxy: " + indyProxyUrl) } @@ -92,7 +92,7 @@ func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo originalIndy := getOriginalIndyBaseUrl(foloTrackContent.Uploads[0].LocalUrl) buildName := common.GenerateRandomBuildName() prev := t - buildSuccess := buildtest.DoRun(originalIndy, indyBaseUrl, indyProxyUrl, packageType, buildName, foloTrackContent, additionalRepos, DEFAULT_ROUTINES, clearCache, dryRun) + buildSuccess := buildtest.DoRun(originalIndy, indyBaseUrl, indyProxyUrl, migrateTargetIndy, packageType, buildName, foloTrackContent, additionalRepos, DEFAULT_ROUTINES, clearCache, dryRun) t = time.Now() fmt.Printf("Create mock group(%s) and download/upload SUCCESS, elapsed(s): %f\n", buildName, t.Sub(prev).Seconds()) diff --git a/pkg/promotetest/run.go b/pkg/promotetest/run.go index af82a6c..4da0635 100644 --- a/pkg/promotetest/run.go +++ b/pkg/promotetest/run.go @@ -46,3 +46,7 @@ func DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum str return promote(indyBaseUrl, foloTrackId, sourceStore, targetStore, paths, dryRun) } + +func MigratePromote(indyURL, trackingId, source, target string, paths []string, dryRun bool) (string, int, bool) { + return promote(indyURL, trackingId, source, target, paths, dryRun) +} \ No newline at end of file From d29ebddc4cb8842edbc348a1858a2a2c17247b91 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Tue, 12 Mar 2024 08:50:11 +0800 Subject: [PATCH 02/18] delete path before uploading, add filter to ignore remote repo --- pkg/buildtest/run.go | 32 +++++++++++--- pkg/integrationtest/run.go | 87 ++++++++++++++++++++------------------ 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index ee01209..80307f1 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -6,6 +6,7 @@ import ( "os" "path" "strings" + "time" "github.com/commonjava/indy-tests/pkg/promotetest" @@ -19,6 +20,8 @@ const ( //migrateTargetIndyHost = "indy-gateway-indy--test.apps.gpc.ocp-hub.prod.psi.redhat.com" ) +const DATA_TIME = "2006-01-02 15:04:05" + func Run(originalIndy, foloId, replacement, targetIndy, packageType string, processNum int) { origIndy := originalIndy if !strings.HasPrefix(origIndy, "http://") { @@ -75,7 +78,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp return success } - migrateFunc := func(md5str, originalArtiURL, targetArtiURL string, migrateTargetArtiURL string) bool { + migrateFunc := func(md5str, originalArtiURL, targetArtiURL, migrateTargetArtiURL string) bool { fileLoc := path.Join(downloadDir, path.Base(targetArtiURL)) if dryRun { fmt.Printf("Dry run download, url: %s\n", targetArtiURL) @@ -102,12 +105,16 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp broken = !common.ConcurrentRun(processNum, downloads, downloadFunc) } else if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) - migrateUploads := prepareDownloadEntriesByFolo(migrateTargetIndyHost, newBuildName, packageType, foloTrackContent, additionalRepos, proxyEnabled) paths := []string{} rhpaths := []string{} for i, down := range downloads { - fmt.Print(i) - broken = !migrateFunc(down[0], down[1], down[2], migrateUploads[i][2]) + if strings.Contains(down[3],"/maven/remote/") || strings.Contains(down[3],"/npm/remote/"){ + continue + } + deletePath := setHostname(down[3],migrateTargetIndyHost) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), deletePath) + broken = !delRequest(deletePath) + broken = !migrateFunc(down[0], down[1], down[2], setHostname(down[2],migrateTargetIndyHost)) if broken { break } else { @@ -122,9 +129,13 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp rhTargetStore := packageType+":hosted:pnc-builds" sourceStore := packageType+":hosted:"+newBuildName promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, targetStore, paths, false) - if rhpaths != nil{ + if len(rhpaths) > 0 { + fmt.Printf("Waiting 180s...\n") + time.Sleep(180 * time.Second) promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, rhTargetStore, rhpaths, false) } + fmt.Println("Purging migate target Hosted and Group repo") + defer DeleteIndyTestRepos("http://"+migrateTargetIndyHost, packageType, newBuildName) } else { for _, down := range downloads { broken = !downloadFunc(down[0], down[1], down[2]) @@ -254,7 +265,7 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string, } downUrl = fmt.Sprintf("%s%s", targetIndy, p) } - result[down.Path] = []string{down.Md5, "", downUrl} + result[down.Path] = []string{down.Md5, "", downUrl, down.LocalUrl} } return result } @@ -326,3 +337,12 @@ func prepareDownUploadDirectories(buildId string, clearCache bool) (string, stri fmt.Printf("Prepared download dir: %s, upload dir: %s\n", downloadDir, uploadDir) return downloadDir, uploadDir } + +func setHostname(addr, hostname string) string { + u, err := url.Parse(addr) + if err != nil { + return "" + } + u.Host = hostname + return u.String() +} diff --git a/pkg/integrationtest/run.go b/pkg/integrationtest/run.go index 45b5543..8f351b7 100644 --- a/pkg/integrationtest/run.go +++ b/pkg/integrationtest/run.go @@ -106,54 +106,57 @@ func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo } } - //f. Retrieve the metadata files which will be affected by promotion - metaFiles := calculateMetadataFiles(foloTrackContent) - metaFilesLoc := path.Join(TMP_METADATA_DIR, "before-promote") - newVersionNum := buildName[len(common.BUILD_TEST_):] - exists := true - passed, e := retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists) - if !passed { - logger.Infof("Metadata validate failed (before). Errors: %s", e.Error()) - panic("Metadata validate failed") - } - fmt.Printf("Metadata validate (before) SUCCESS\n") - - //g. Promote the files in hosted repo A to hosted repo pnc-builds - foloTrackId := buildName - sourceStore, targetStore := getPromotionSrcTargetStores(packageType, buildName, promoteTargetStore, foloTrackContent) - resp, _, success := promotetest.DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum, foloTrackContent, dryRun) - if !success { - fmt.Printf("Promote failed, %s\n", resp) - panic("Promote failed") - } + migrateEnabled := (migrateTargetIndy != "") + if !migrateEnabled { + //f. Retrieve the metadata files which will be affected by promotion + metaFiles := calculateMetadataFiles(foloTrackContent) + metaFilesLoc := path.Join(TMP_METADATA_DIR, "before-promote") + newVersionNum := buildName[len(common.BUILD_TEST_):] + exists := true + passed, e := retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists) + if !passed { + logger.Infof("Metadata validate failed (before). Errors: %s", e.Error()) + panic("Metadata validate failed") + } + fmt.Printf("Metadata validate (before) SUCCESS\n") + + //g. Promote the files in hosted repo A to hosted repo pnc-builds + foloTrackId := buildName + sourceStore, targetStore := getPromotionSrcTargetStores(packageType, buildName, promoteTargetStore, foloTrackContent) + resp, _, success := promotetest.DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum, foloTrackContent, dryRun) + if !success { + fmt.Printf("Promote failed, %s\n", resp) + panic("Promote failed") + } - //h. Retrieve the metadata files again, check the new version - fmt.Printf("Waiting 30s...\n") - time.Sleep(30 * time.Second) // wait for Indy event handled + //h. Retrieve the metadata files again, check the new version + fmt.Printf("Waiting 30s...\n") + time.Sleep(30 * time.Second) // wait for Indy event handled - metaFilesLoc = path.Join(TMP_METADATA_DIR, "after-promote") - passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, exists) - if !passed { - logger.Infof("Metadata validate failed (after promotion). Errors: %s", e.Error()) - panic("Metadata validate failed") - } - fmt.Printf("Metadata validate (after promotion) SUCCESS\n") + metaFilesLoc = path.Join(TMP_METADATA_DIR, "after-promote") + passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, exists) + if !passed { + logger.Infof("Metadata validate failed (after promotion). Errors: %s", e.Error()) + panic("Metadata validate failed") + } + fmt.Printf("Metadata validate (after promotion) SUCCESS\n") - //i. Rollback the promotion - fmt.Printf("Rollback:\n%s\n", resp) - promotetest.Rollback(indyBaseUrl, resp, dryRun) + //i. Rollback the promotion + fmt.Printf("Rollback:\n%s\n", resp) + promotetest.Rollback(indyBaseUrl, resp, dryRun) - //h. Retrieve the metadata files again, check the new version is GONE - fmt.Printf("Waiting 30s...\n") - time.Sleep(30 * time.Second) + //h. Retrieve the metadata files again, check the new version is GONE + fmt.Printf("Waiting 30s...\n") + time.Sleep(30 * time.Second) - metaFilesLoc = path.Join(TMP_METADATA_DIR, "rollback") - passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists) - if !passed { - logger.Infof("Metadata validate failed (rollback). Errors: %s", e.Error()) - panic("Metadata validate failed") + metaFilesLoc = path.Join(TMP_METADATA_DIR, "rollback") + passed, e = retrieveMetadataAndValidate(indyBaseUrl, packageType, metaCheckRepo, metaFiles, metaFilesLoc, newVersionNum, !exists) + if !passed { + logger.Infof("Metadata validate failed (rollback). Errors: %s", e.Error()) + panic("Metadata validate failed") + } + fmt.Printf("Metadata validate (rollback) SUCCESS\n") } - fmt.Printf("Metadata validate (rollback) SUCCESS\n") // Pause and keep pod for debugging if keepPod { From 3aa8d270cb0dd7dbdfdcdf891497ea3697d8dac2 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Tue, 12 Mar 2024 08:54:51 +0800 Subject: [PATCH 03/18] format code --- pkg/buildtest/run.go | 26 +++++++++++++------------- pkg/promotetest/run.go | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 80307f1..036c3b9 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -108,26 +108,26 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp paths := []string{} rhpaths := []string{} for i, down := range downloads { - if strings.Contains(down[3],"/maven/remote/") || strings.Contains(down[3],"/npm/remote/"){ + if strings.Contains(down[3], "/maven/remote/") || strings.Contains(down[3], "/npm/remote/") { continue } - deletePath := setHostname(down[3],migrateTargetIndyHost) + deletePath := setHostname(down[3], migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), deletePath) broken = !delRequest(deletePath) - broken = !migrateFunc(down[0], down[1], down[2], setHostname(down[2],migrateTargetIndyHost)) + broken = !migrateFunc(down[0], down[1], down[2], setHostname(down[2], migrateTargetIndyHost)) if broken { break } else { - if strings.Contains(i, "redhat"){ + if strings.Contains(i, "redhat") { rhpaths = append(rhpaths, i) } else { paths = append(paths, i) } } } - targetStore := packageType+":hosted:shared-imports" - rhTargetStore := packageType+":hosted:pnc-builds" - sourceStore := packageType+":hosted:"+newBuildName + targetStore := packageType + ":hosted:shared-imports" + rhTargetStore := packageType + ":hosted:pnc-builds" + sourceStore := packageType + ":hosted:" + newBuildName promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, targetStore, paths, false) if len(rhpaths) > 0 { fmt.Printf("Waiting 180s...\n") @@ -339,10 +339,10 @@ func prepareDownUploadDirectories(buildId string, clearCache bool) (string, stri } func setHostname(addr, hostname string) string { - u, err := url.Parse(addr) - if err != nil { - return "" - } - u.Host = hostname - return u.String() + u, err := url.Parse(addr) + if err != nil { + return "" + } + u.Host = hostname + return u.String() } diff --git a/pkg/promotetest/run.go b/pkg/promotetest/run.go index 4da0635..47b18fc 100644 --- a/pkg/promotetest/run.go +++ b/pkg/promotetest/run.go @@ -49,4 +49,4 @@ func DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum str func MigratePromote(indyURL, trackingId, source, target string, paths []string, dryRun bool) (string, int, bool) { return promote(indyURL, trackingId, source, target, paths, dryRun) -} \ No newline at end of file +} From 65310fe909bd7b7d08fd2f786fc5a7b7ccb90a30 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Wed, 13 Mar 2024 09:28:37 +0800 Subject: [PATCH 04/18] use direct PUT instead --- pkg/buildtest/run.go | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 036c3b9..5d2eec1 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -8,8 +8,6 @@ import ( "strings" "time" - "github.com/commonjava/indy-tests/pkg/promotetest" - common "github.com/commonjava/indy-tests/pkg/common" ) @@ -17,7 +15,6 @@ const ( TMP_DOWNLOAD_DIR = "/tmp/download" TMP_UPLOAD_DIR = "/tmp/upload" PROXY_ = "proxy-" - //migrateTargetIndyHost = "indy-gateway-indy--test.apps.gpc.ocp-hub.prod.psi.redhat.com" ) const DATA_TIME = "2006-01-02 15:04:05" @@ -50,7 +47,6 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) fmt.Printf("Migrate to host %s", migrateTargetIndyHost) - prepareIndyRepos("http://"+migrateTargetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) } trackingId := foloTrackContent.TrackingKey.Id @@ -105,37 +101,18 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp broken = !common.ConcurrentRun(processNum, downloads, downloadFunc) } else if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) - paths := []string{} - rhpaths := []string{} - for i, down := range downloads { + for _, down := range downloads { if strings.Contains(down[3], "/maven/remote/") || strings.Contains(down[3], "/npm/remote/") { continue } - deletePath := setHostname(down[3], migrateTargetIndyHost) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), deletePath) - broken = !delRequest(deletePath) - broken = !migrateFunc(down[0], down[1], down[2], setHostname(down[2], migrateTargetIndyHost)) + migratePath := setHostname(down[3], migrateTargetIndyHost) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) + broken = !delRequest(migratePath) + broken = !migrateFunc(down[0], down[1], down[2], migratePath) if broken { break - } else { - if strings.Contains(i, "redhat") { - rhpaths = append(rhpaths, i) - } else { - paths = append(paths, i) - } } } - targetStore := packageType + ":hosted:shared-imports" - rhTargetStore := packageType + ":hosted:pnc-builds" - sourceStore := packageType + ":hosted:" + newBuildName - promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, targetStore, paths, false) - if len(rhpaths) > 0 { - fmt.Printf("Waiting 180s...\n") - time.Sleep(180 * time.Second) - promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, rhTargetStore, rhpaths, false) - } - fmt.Println("Purging migate target Hosted and Group repo") - defer DeleteIndyTestRepos("http://"+migrateTargetIndyHost, packageType, newBuildName) } else { for _, down := range downloads { broken = !downloadFunc(down[0], down[1], down[2]) From aa428bcab173e38505c942c44a1e74e073384e85 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Thu, 14 Mar 2024 16:30:48 +0800 Subject: [PATCH 05/18] moved migration function out of the download, add remote handling for 3 path --- pkg/buildtest/run.go | 89 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 5d2eec1..5fb00d1 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -74,7 +74,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp return success } - migrateFunc := func(md5str, originalArtiURL, targetArtiURL, migrateTargetArtiURL string) bool { + migrateFunc := func(md5str, targetArtiURL, migrateTargetArtiURL string) bool { fileLoc := path.Join(downloadDir, path.Base(targetArtiURL)) if dryRun { fmt.Printf("Dry run download, url: %s\n", targetArtiURL) @@ -94,25 +94,43 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp } broken := false - if len(downloads) > 0 { + + if migrateEnabled { + migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) + migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, newBuildName, packageType, foloTrackContent, additionalRepos, proxyEnabled) + for _, down := range migrateArtifacts { + migratePath := setHostname(down[3], migrateTargetIndyHost) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) + broken = !delRequest(migratePath) + if strings.Contains(down[2], "npm:remote:npmjs") { + migratePath := strings.Replace(migratePath, "/npm/remote/npmjs", "/npm/hosted/shared-imports", 1) + broken = !delRequest(migratePath) + } else if strings.Contains(down[2], "maven:remote:central") { + migratePath := strings.Replace(migratePath, "/maven/remote/central", "/maven/hosted/shared-imports", 1) + broken = !delRequest(migratePath) + } else if strings.Contains(down[2], "maven:remote:mrrc-ga-rh") { + migratePath := strings.Replace(migratePath, "/maven/remote/mrrc-ga-rh", "/maven/hosted/pnc-builds", 1) + broken = !delRequest(migratePath) + } + broken = !migrateFunc(down[0], down[2], migratePath) + if broken { + break + } + } + fmt.Println("==========================================") + if broken { + fmt.Printf("Build test failed due to some downloading errors. Please see above logs to see the details.\n\n") + os.Exit(1) + } + fmt.Printf("Migration artifacts handling finished.\n\n") + return true + } + + if len(downloads) > 0 && !migrateEnabled { fmt.Println("Start handling downloads artifacts.") fmt.Printf("==========================================\n\n") - if processNum > 1 && !migrateEnabled { + if processNum > 1 { broken = !common.ConcurrentRun(processNum, downloads, downloadFunc) - } else if migrateEnabled { - migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) - for _, down := range downloads { - if strings.Contains(down[3], "/maven/remote/") || strings.Contains(down[3], "/npm/remote/") { - continue - } - migratePath := setHostname(down[3], migrateTargetIndyHost) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) - broken = !migrateFunc(down[0], down[1], down[2], migratePath) - if broken { - break - } - } } else { for _, down := range downloads { broken = !downloadFunc(down[0], down[1], down[2]) @@ -152,7 +170,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp uploads := prepareUploadEntriesByFolo(originalIndy, targetIndy, newBuildName, foloTrackContent) - if len(uploads) > 0 { + if len(uploads) > 0 && !migrateEnabled { fmt.Println("Start handling uploads artifacts.") fmt.Printf("==========================================\n\n") if processNum > 1 { @@ -242,7 +260,40 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string, } downUrl = fmt.Sprintf("%s%s", targetIndy, p) } - result[down.Path] = []string{down.Md5, "", downUrl, down.LocalUrl} + result[down.Path] = []string{down.Md5, "", downUrl} + } + return result +} + +func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, + foloRecord common.TrackedContent, additionalRepos []string, proxyEnabled bool) map[string][]string { + targetIndy := normIndyURL(targetIndyURL) + result := make(map[string][]string) + for _, down := range foloRecord.Downloads { + var p string + downUrl := "" + repoPath := strings.ReplaceAll(down.StoreKey, ":", "/") + if down.AccessChannel == "GENERIC_PROXY" { + if proxyEnabled { + downUrl = fmt.Sprintf("%s%s", PROXY_, down.OriginUrl) + } else { + // Generic remote repo may not be available when we replay the build. PNC has promoted the files to "h-" repo, + // so we replace "generic-http/remote/r-xxxx" to "generic-http:hosted:h-xxxx" + repoPath = strings.Replace(repoPath, "generic-http/remote/r-", "generic-http/hosted/h-", 1) + p = path.Join("api/content", repoPath, down.Path) + downUrl = fmt.Sprintf("%s%s", targetIndy, p) + } + } else { + // To explain the 'HasPrefix': NPM build can have downloads from maven repos (or vice verse). We use the original repo + // if the storeKey is not compliant with packageType + if common.Contains(additionalRepos, down.StoreKey) || !strings.HasPrefix(down.StoreKey, packageType) { + p = path.Join("api/folo/track", newBuildId, repoPath, down.Path) + } else { + p = path.Join("api/folo/track", newBuildId, packageType, "group", newBuildId, down.Path) + } + downUrl = fmt.Sprintf("%s%s", targetIndy, p) + } + result[down.Path] = []string{down.Md5, repoPath, downUrl, down.LocalUrl} } return result } From ac5017ace88f85ac4afe88eecd654621a71b8f3a Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Thu, 14 Mar 2024 20:41:38 +0800 Subject: [PATCH 06/18] add more deletion to ensure --- pkg/buildtest/run.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 5fb00d1..d310c91 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -102,15 +102,27 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp migratePath := setHostname(down[3], migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) - if strings.Contains(down[2], "npm:remote:npmjs") { + + if down[1] != packageType+":hosted:shared-imports" { + extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) + broken = !delRequest(extra) + } + + if down[1] == "npm:remote:npmjs" { migratePath := strings.Replace(migratePath, "/npm/remote/npmjs", "/npm/hosted/shared-imports", 1) broken = !delRequest(migratePath) - } else if strings.Contains(down[2], "maven:remote:central") { + + } else if down[1] == "maven:remote:central" { migratePath := strings.Replace(migratePath, "/maven/remote/central", "/maven/hosted/shared-imports", 1) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) - } else if strings.Contains(down[2], "maven:remote:mrrc-ga-rh") { + + } else if down[1] == "maven:remote:mrrc-ga-rh" { migratePath := strings.Replace(migratePath, "/maven/remote/mrrc-ga-rh", "/maven/hosted/pnc-builds", 1) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) + } broken = !migrateFunc(down[0], down[2], migratePath) if broken { @@ -293,7 +305,7 @@ func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, } downUrl = fmt.Sprintf("%s%s", targetIndy, p) } - result[down.Path] = []string{down.Md5, repoPath, downUrl, down.LocalUrl} + result[down.Path] = []string{down.Md5, down.StoreKey, downUrl, down.LocalUrl, down.Path} } return result } From a27509fdb0217996cd63d2d70ceec3905667574c Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Thu, 14 Mar 2024 22:28:25 +0800 Subject: [PATCH 07/18] changed migate download path --- pkg/buildtest/run.go | 49 ++++++++++++++++---------------------- pkg/integrationtest/run.go | 14 +++++------ 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index d310c91..8a0781b 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -37,16 +37,16 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp common.ValidateTargetIndyOrExit(originalIndy) targetIndyHost, _ := common.ValidateTargetIndyOrExit(targetIndy) - // Prepare the indy repos for the whole testing - buildMeta := decideMeta(packageType) - if !prepareIndyRepos("http://"+targetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) { - os.Exit(1) - } - migrateEnabled := (migrateTargetIndy != "") if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) fmt.Printf("Migrate to host %s", migrateTargetIndyHost) + } else { + // Prepare the indy repos for the whole testing + buildMeta := decideMeta(packageType) + if !prepareIndyRepos("http://"+targetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) { + os.Exit(1) + } } trackingId := foloTrackContent.TrackingKey.Id @@ -97,7 +97,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) - migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, newBuildName, packageType, foloTrackContent, additionalRepos, proxyEnabled) + migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, newBuildName, packageType, foloTrackContent) for _, down := range migrateArtifacts { migratePath := setHostname(down[3], migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) @@ -109,17 +109,18 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp broken = !delRequest(extra) } - if down[1] == "npm:remote:npmjs" { - migratePath := strings.Replace(migratePath, "/npm/remote/npmjs", "/npm/hosted/shared-imports", 1) + if down[1] == "npm:remote:npmjs" || down[1] == "maven:remote:central" { + migratePath := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) - } else if down[1] == "maven:remote:central" { - migratePath := strings.Replace(migratePath, "/maven/remote/central", "/maven/hosted/shared-imports", 1) + } else if down[1] == "maven:remote:mrrc-ga-rh" { + migratePath := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) - } else if down[1] == "maven:remote:mrrc-ga-rh" { - migratePath := strings.Replace(migratePath, "/maven/remote/mrrc-ga-rh", "/maven/hosted/pnc-builds", 1) + } else if stings.HasPrefix(down[1], "maven:hosted:build-") { + migratePath := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) @@ -278,7 +279,7 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string, } func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, - foloRecord common.TrackedContent, additionalRepos []string, proxyEnabled bool) map[string][]string { + foloRecord common.TrackedContent) map[string][]string { targetIndy := normIndyURL(targetIndyURL) result := make(map[string][]string) for _, down := range foloRecord.Downloads { @@ -286,22 +287,14 @@ func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, downUrl := "" repoPath := strings.ReplaceAll(down.StoreKey, ":", "/") if down.AccessChannel == "GENERIC_PROXY" { - if proxyEnabled { - downUrl = fmt.Sprintf("%s%s", PROXY_, down.OriginUrl) - } else { - // Generic remote repo may not be available when we replay the build. PNC has promoted the files to "h-" repo, - // so we replace "generic-http/remote/r-xxxx" to "generic-http:hosted:h-xxxx" - repoPath = strings.Replace(repoPath, "generic-http/remote/r-", "generic-http/hosted/h-", 1) - p = path.Join("api/content", repoPath, down.Path) - downUrl = fmt.Sprintf("%s%s", targetIndy, p) - } + repoPath = strings.Replace(repoPath, "generic-http/remote/r-", "generic-http/hosted/h-", 1) + p = path.Join("api/content", repoPath, down.Path) + downUrl = fmt.Sprintf("%s%s", targetIndy, p) } else { - // To explain the 'HasPrefix': NPM build can have downloads from maven repos (or vice verse). We use the original repo - // if the storeKey is not compliant with packageType - if common.Contains(additionalRepos, down.StoreKey) || !strings.HasPrefix(down.StoreKey, packageType) { - p = path.Join("api/folo/track", newBuildId, repoPath, down.Path) + if !strings.HasPrefix(down.StoreKey, packageType) { + p = path.Join("api/content", repoPath, down.Path) } else { - p = path.Join("api/folo/track", newBuildId, packageType, "group", newBuildId, down.Path) + p = path.Join("api/content", packageType, "group/builds-untested+shared-imports+public", down.Path) } downUrl = fmt.Sprintf("%s%s", targetIndy, p) } diff --git a/pkg/integrationtest/run.go b/pkg/integrationtest/run.go index 8f351b7..994c6a3 100644 --- a/pkg/integrationtest/run.go +++ b/pkg/integrationtest/run.go @@ -99,15 +99,15 @@ func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo //k. Delete the temp group and the hosted repo, and folo record defer cleanUp(indyBaseUrl, packageType, buildName, dryRun) - // Advanced checks - if buildSuccess && !dryRun { - if !verifyFoloRecord(indyBaseUrl, buildName, foloTrackContent) { - return - } - } - migrateEnabled := (migrateTargetIndy != "") if !migrateEnabled { + // Advanced checks + if buildSuccess && !dryRun { + if !verifyFoloRecord(indyBaseUrl, buildName, foloTrackContent) { + return + } + } + //f. Retrieve the metadata files which will be affected by promotion metaFiles := calculateMetadataFiles(foloTrackContent) metaFilesLoc := path.Join(TMP_METADATA_DIR, "before-promote") From edcb244bcdafa789113aee5cbc93ea53eba59799 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Thu, 14 Mar 2024 22:30:59 +0800 Subject: [PATCH 08/18] fix typo --- pkg/buildtest/run.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 8a0781b..3354ff8 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -110,17 +110,17 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp } if down[1] == "npm:remote:npmjs" || down[1] == "maven:remote:central" { - migratePath := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) + migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) } else if down[1] == "maven:remote:mrrc-ga-rh" { - migratePath := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) + migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) - } else if stings.HasPrefix(down[1], "maven:hosted:build-") { - migratePath := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) + } else if strings.HasPrefix(down[1], "maven:hosted:build-") { + migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) From 33fa603730707d41ba1d80d4ecd6613ab94d5d74 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Thu, 14 Mar 2024 23:40:41 +0800 Subject: [PATCH 09/18] switched repo --- pkg/buildtest/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 3354ff8..c5bc4de 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -294,7 +294,7 @@ func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, if !strings.HasPrefix(down.StoreKey, packageType) { p = path.Join("api/content", repoPath, down.Path) } else { - p = path.Join("api/content", packageType, "group/builds-untested+shared-imports+public", down.Path) + p = path.Join("api/content", packageType, "group/build-test-91388", down.Path) } downUrl = fmt.Sprintf("%s%s", targetIndy, p) } From d106a42413b6a810d3b70009dc4d583bd0bcf0e8 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Fri, 15 Mar 2024 00:38:59 +0800 Subject: [PATCH 10/18] disable deletion for now --- pkg/buildtest/run.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index c5bc4de..7ccecdc 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -101,28 +101,23 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp for _, down := range migrateArtifacts { migratePath := setHostname(down[3], migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) + //broken = !delRequest(migratePath) if down[1] != packageType+":hosted:shared-imports" { extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) - broken = !delRequest(extra) + //broken = !delRequest(extra) } if down[1] == "npm:remote:npmjs" || down[1] == "maven:remote:central" { migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) + //broken = !delRequest(migratePath) - } else if down[1] == "maven:remote:mrrc-ga-rh" { + } else if down[1] == "maven:remote:mrrc-ga-rh" || strings.HasPrefix(down[1], "maven:hosted:build-") { migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) - - } else if strings.HasPrefix(down[1], "maven:hosted:build-") { - migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) + //broken = !delRequest(migratePath) } broken = !migrateFunc(down[0], down[2], migratePath) From 16aa508bbb4db3ee92884e761c359af3dd497f7e Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Fri, 15 Mar 2024 01:01:22 +0800 Subject: [PATCH 11/18] moved deletion out for consistency --- pkg/buildtest/run.go | 61 ++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 7ccecdc..b89d7b7 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -97,30 +97,11 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) - migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, newBuildName, packageType, foloTrackContent) + migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, migrateTargetIndyHost, packageType, foloTrackContent) + fmt.Printf("Waiting 60s...\n") + time.Sleep(60 * time.Second) // wait for Indy event handled for _, down := range migrateArtifacts { - migratePath := setHostname(down[3], migrateTargetIndyHost) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - //broken = !delRequest(migratePath) - - if down[1] != packageType+":hosted:shared-imports" { - extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) - //broken = !delRequest(extra) - } - - if down[1] == "npm:remote:npmjs" || down[1] == "maven:remote:central" { - migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down[4]) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - //broken = !delRequest(migratePath) - - } else if down[1] == "maven:remote:mrrc-ga-rh" || strings.HasPrefix(down[1], "maven:hosted:build-") { - migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down[4]) - fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - //broken = !delRequest(migratePath) - - } - broken = !migrateFunc(down[0], down[2], migratePath) + broken = !migrateFunc(down[0], down[2], down[1]) if broken { break } @@ -273,7 +254,7 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string, return result } -func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, +func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageType string, foloRecord common.TrackedContent) map[string][]string { targetIndy := normIndyURL(targetIndyURL) result := make(map[string][]string) @@ -284,16 +265,42 @@ func prepareMigrateEntriesByFolo(targetIndyURL, newBuildId, packageType string, if down.AccessChannel == "GENERIC_PROXY" { repoPath = strings.Replace(repoPath, "generic-http/remote/r-", "generic-http/hosted/h-", 1) p = path.Join("api/content", repoPath, down.Path) - downUrl = fmt.Sprintf("%s%s", targetIndy, p) } else { if !strings.HasPrefix(down.StoreKey, packageType) { p = path.Join("api/content", repoPath, down.Path) } else { p = path.Join("api/content", packageType, "group/build-test-91388", down.Path) } - downUrl = fmt.Sprintf("%s%s", targetIndy, p) } - result[down.Path] = []string{down.Md5, down.StoreKey, downUrl, down.LocalUrl, down.Path} + + downUrl = fmt.Sprintf("%s%s", targetIndy, p) + + broken := false + migratePath := setHostname(downUrl, migrateTargetIndyHost) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) + broken = !delRequest(migratePath) + + if down.StoreKey != packageType+":hosted:shared-imports" { + extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) + broken = !delRequest(extra) + } + + if down.StoreKey == "npm:remote:npmjs" || down.StoreKey == "maven:remote:central" { + migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) + broken = !delRequest(migratePath) + } else if down.StoreKey == "maven:remote:mrrc-ga-rh" || strings.HasPrefix(down.StoreKey, "maven:hosted:build-") { + migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down.Path) + fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) + broken = !delRequest(migratePath) + } + + if broken { + fmt.Printf("[%s] Deletion failed for %s\n", time.Now().Format(DATA_TIME), migratePath) + } + + result[down.Path] = []string{down.Md5, migratePath, downUrl} } return result } From abfd41024a68f97242e762c296bebbe9cdab3125 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Fri, 15 Mar 2024 01:03:44 +0800 Subject: [PATCH 12/18] fix typo --- pkg/buildtest/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index b89d7b7..8262ad0 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -276,7 +276,7 @@ func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageTy downUrl = fmt.Sprintf("%s%s", targetIndy, p) broken := false - migratePath := setHostname(downUrl, migrateTargetIndyHost) + migratePath := setHostname(down.LocalUrl, migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delRequest(migratePath) From 47e68206e15463b1617779c29861816486b02889 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Fri, 15 Mar 2024 01:22:27 +0800 Subject: [PATCH 13/18] fix scope problem --- pkg/buildtest/run.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 8262ad0..7acdc9a 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -101,7 +101,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp fmt.Printf("Waiting 60s...\n") time.Sleep(60 * time.Second) // wait for Indy event handled for _, down := range migrateArtifacts { - broken = !migrateFunc(down[0], down[2], down[1]) + broken = !migrateFunc(down[0], down[1], down[2]) if broken { break } @@ -278,29 +278,29 @@ func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageTy broken := false migratePath := setHostname(down.LocalUrl, migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) + broken = !delArtifact(migratePath) if down.StoreKey != packageType+":hosted:shared-imports" { extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) - broken = !delRequest(extra) + broken = !delArtifact(extra) } if down.StoreKey == "npm:remote:npmjs" || down.StoreKey == "maven:remote:central" { - migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) + migratePath, _ = url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) + broken = !delArtifact(migratePath) } else if down.StoreKey == "maven:remote:mrrc-ga-rh" || strings.HasPrefix(down.StoreKey, "maven:hosted:build-") { - migratePath, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down.Path) + migratePath, _ = url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) - broken = !delRequest(migratePath) + broken = !delArtifact(migratePath) } if broken { fmt.Printf("[%s] Deletion failed for %s\n", time.Now().Format(DATA_TIME), migratePath) } - result[down.Path] = []string{down.Md5, migratePath, downUrl} + result[down.Path] = []string{down.Md5, downUrl, migratePath} } return result } @@ -381,3 +381,8 @@ func setHostname(addr, hostname string) string { u.Host = hostname return u.String() } + +func delArtifact(url string) bool { + _, _, succeeded := common.HTTPRequest(url, common.MethodDelete, nil, false, nil, nil, "", false) + return succeeded +} From 399f6850b514d36ed18d6279eb0b0860e6a6816e Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Fri, 15 Mar 2024 01:35:14 +0800 Subject: [PATCH 14/18] replaced group with realtime name --- pkg/buildtest/run.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 7acdc9a..3dd8b11 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -41,12 +41,11 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) fmt.Printf("Migrate to host %s", migrateTargetIndyHost) - } else { - // Prepare the indy repos for the whole testing - buildMeta := decideMeta(packageType) - if !prepareIndyRepos("http://"+targetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) { - os.Exit(1) - } + } + // Prepare the indy repos for the whole testing + buildMeta := decideMeta(packageType) + if !prepareIndyRepos("http://"+targetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun) { + os.Exit(1) } trackingId := foloTrackContent.TrackingKey.Id @@ -97,7 +96,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp if migrateEnabled { migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) - migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, migrateTargetIndyHost, packageType, foloTrackContent) + migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, migrateTargetIndyHost, packageType, newBuildName, foloTrackContent) fmt.Printf("Waiting 60s...\n") time.Sleep(60 * time.Second) // wait for Indy event handled for _, down := range migrateArtifacts { @@ -255,7 +254,7 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string, } func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageType string, - foloRecord common.TrackedContent) map[string][]string { + newBuildId, foloRecord common.TrackedContent) map[string][]string { targetIndy := normIndyURL(targetIndyURL) result := make(map[string][]string) for _, down := range foloRecord.Downloads { @@ -269,7 +268,7 @@ func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageTy if !strings.HasPrefix(down.StoreKey, packageType) { p = path.Join("api/content", repoPath, down.Path) } else { - p = path.Join("api/content", packageType, "group/build-test-91388", down.Path) + p = path.Join("api/content", packageType, "group", newBuildId, down.Path) } } From 7c507667fcc30de239a1c7e8c2d0d0530e388d73 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Fri, 15 Mar 2024 01:37:48 +0800 Subject: [PATCH 15/18] fix typo --- pkg/buildtest/run.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 3dd8b11..9ae63c6 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -253,8 +253,8 @@ func prepareDownloadEntriesByFolo(targetIndyURL, newBuildId, packageType string, return result } -func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageType string, - newBuildId, foloRecord common.TrackedContent) map[string][]string { +func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageType, + newBuildId string, foloRecord common.TrackedContent) map[string][]string { targetIndy := normIndyURL(targetIndyURL) result := make(map[string][]string) for _, down := range foloRecord.Downloads { From cb62ee874256bb9543515e3327e24b48a4540bdc Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Tue, 19 Mar 2024 14:23:20 +0800 Subject: [PATCH 16/18] moved to dedicate command for migration --- cmd/integrationtest/command.go | 4 +-- cmd/migrate/command.go | 59 ++++++++++++++++++++++++++++++++++ cmd/root/main.go | 2 ++ 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 cmd/migrate/command.go diff --git a/cmd/integrationtest/command.go b/cmd/integrationtest/command.go index a719a42..c5cc1a8 100644 --- a/cmd/integrationtest/command.go +++ b/cmd/integrationtest/command.go @@ -40,12 +40,11 @@ func NewIntegrationTestCmd() *cobra.Command { keepPod, _ := cmd.Flags().GetBool("keepPod") sidecar, _ := cmd.Flags().GetBool("sidecar") indyProxyUrl, _ := cmd.Flags().GetString("indyProxyUrl") - migrateTargetIndy, _ := cmd.Flags().GetString("migrateTargetIndy") metaCheckRepo := "" if len(args) >= 5 { metaCheckRepo = args[4] } - integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl, migrateTargetIndy) + integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl, "") }, } @@ -54,7 +53,6 @@ func NewIntegrationTestCmd() *cobra.Command { exec.Flags().BoolP("keepPod", "k", false, "Keep the pod after test to debug.") exec.Flags().BoolP("sidecar", "s", false, "Send requests through sidecar.") exec.Flags().StringP("indyProxyUrl", "p", "", "Indy generic proxy url.") - exec.Flags().StringP("migrateTargetIndy", "m", "", "Migrate download artifacts to new Indy url") return exec } diff --git a/cmd/migrate/command.go b/cmd/migrate/command.go new file mode 100644 index 0000000..97fdbaa --- /dev/null +++ b/cmd/migrate/command.go @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2021-2023 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package migrate + + import ( + "fmt" + "os" + + "github.com/commonjava/indy-tests/pkg/integrationtest" + "github.com/spf13/cobra" + ) + + func NewMigrateCmd() *cobra.Command { + + exec := &cobra.Command{ + Use: "migrate $indyBaseUrl $datasetRepoUrl $buildId $migrateTargetIndy --dryRun(optional)", + Short: "To migrate artifact", + Example: "integrationtest http://indy.xyz.com https://gitlab.xyz.com/nos/nos-integrationtest-dataset 2836 test-builds", + Run: func(cmd *cobra.Command, args []string) { + if !validate(args) { + cmd.Help() + os.Exit(1) + } + clearCache, _ := cmd.Flags().GetBool("clearCache") + dryRun, _ := cmd.Flags().GetBool("dryRun") + keepPod, _ := cmd.Flags().GetBool("keepPod") + promoteTargetStore := "pnc-builds" + integrationtest.Run(args[0], args[1], args[2], promoteTargetStore, "", clearCache, dryRun, keepPod, false, "", args[3]) + }, + } + + exec.Flags().BoolP("clearCache", "c", false, "Clear cached built artifact files. This will force download from origin again.") + exec.Flags().BoolP("dryRun", "d", false, "Print msg for repo creation, down/upload, promote, and clean up, without really doing it.") + exec.Flags().BoolP("keepPod", "k", false, "Keep the pod after test to debug.") + return exec + } + + func validate(args []string) bool { + if len(args) < 4 { + fmt.Printf("There are 4 mandatory arguments: indyBaseUrl, datasetRepoUrl, buildId, migrateTargetIndy!\n") + return false + } + return true + } + \ No newline at end of file diff --git a/cmd/root/main.go b/cmd/root/main.go index 59a0aa9..d0f8692 100644 --- a/cmd/root/main.go +++ b/cmd/root/main.go @@ -11,6 +11,7 @@ import ( "github.com/commonjava/indy-tests/cmd/integrationtest" "github.com/commonjava/indy-tests/cmd/promotetest" "github.com/commonjava/indy-tests/cmd/statictest" + "github.com/commonjava/indy-tests/cmd/migrate" "github.com/spf13/cobra" ) @@ -29,6 +30,7 @@ func main() { rootCmd.AddCommand(integrationtest.NewIntegrationTestCmd()) rootCmd.AddCommand(event.NewEventTestCmd()) rootCmd.AddCommand(statictest.NewStaticTestCmd()) + rootCmd.AddCommand(migrate.NewMigrateCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) From 3915989ea294924e2958d6deec6e57d9e9df1a63 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Tue, 19 Mar 2024 15:27:34 +0800 Subject: [PATCH 17/18] adds small time gap for deleting artifacts, because of inconsistency problem --- pkg/buildtest/run.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index 9ae63c6..b717829 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -98,7 +98,7 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageTyp migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy) migrateArtifacts := prepareMigrateEntriesByFolo(targetIndy, migrateTargetIndyHost, packageType, newBuildName, foloTrackContent) fmt.Printf("Waiting 60s...\n") - time.Sleep(60 * time.Second) // wait for Indy event handled + time.Sleep(120 * time.Second) // wait for Indy event handled for _, down := range migrateArtifacts { broken = !migrateFunc(down[0], down[1], down[2]) if broken { @@ -278,21 +278,25 @@ func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageTy migratePath := setHostname(down.LocalUrl, migrateTargetIndyHost) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delArtifact(migratePath) + time.Sleep(100 * time.Millisecond) - if down.StoreKey != packageType+":hosted:shared-imports" { + if strings.HasSuffix( down.StoreKey, ":hosted:shared-imports" ) { extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) broken = !delArtifact(extra) + time.Sleep(100 * time.Millisecond) } if down.StoreKey == "npm:remote:npmjs" || down.StoreKey == "maven:remote:central" { migratePath, _ = url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delArtifact(migratePath) + time.Sleep(100 * time.Millisecond) } else if down.StoreKey == "maven:remote:mrrc-ga-rh" || strings.HasPrefix(down.StoreKey, "maven:hosted:build-") { migratePath, _ = url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/pnc-builds", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), migratePath) broken = !delArtifact(migratePath) + time.Sleep(100 * time.Millisecond) } if broken { From 8a5ccfb984a11a95d533b54c2444059e96321086 Mon Sep 17 00:00:00 2001 From: shokakucarrier Date: Tue, 19 Mar 2024 16:44:38 +0800 Subject: [PATCH 18/18] fix typo --- pkg/buildtest/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/buildtest/run.go b/pkg/buildtest/run.go index b717829..0145f72 100644 --- a/pkg/buildtest/run.go +++ b/pkg/buildtest/run.go @@ -280,7 +280,7 @@ func prepareMigrateEntriesByFolo(targetIndyURL, migrateTargetIndyHost, packageTy broken = !delArtifact(migratePath) time.Sleep(100 * time.Millisecond) - if strings.HasSuffix( down.StoreKey, ":hosted:shared-imports" ) { + if !strings.HasSuffix( down.StoreKey, ":hosted:shared-imports" ) { extra, _ := url.JoinPath("http://"+migrateTargetIndyHost, "/api/content", packageType, "/hosted/shared-imports", down.Path) fmt.Printf("[%s] Deleting %s\n", time.Now().Format(DATA_TIME), extra) broken = !delArtifact(extra)