Skip to content

Commit 9f9ae6c

Browse files
sprucespruce
authored
Merge pull request #251 from kathrin-7978/master
Implement CSV-based batch migration with per-project repository mapping
2 parents a84b96b + af40b31 commit 9f9ae6c

File tree

5 files changed

+174
-21
lines changed

5 files changed

+174
-21
lines changed

‎README.md‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ Go to [Settings / Access Tokens](https://gitlab.com/-/profile/personal_access_to
8282

8383
#### gitlab.projectID
8484

85-
Leave it null for the first run of the script. Then the script will show you which projects there are. Can be either string or number.
85+
Leave it null for the first run of the script (leave csvImport.projectMapCsv empty either). Then the script will show you which projects there are. Can be either string or number.
86+
Leave it null if you want to import multiple projects via csv file (set `csvImport` instead).
8687

8788
#### gitlab.listArchivedProjects
8889

@@ -156,6 +157,23 @@ Maps the usernames from gitlab to github. If the assinee of the gitlab issue is
156157

157158
When one renames the project while transfering so that the projects don't loose there links to the mentioned issues.
158159

160+
### csvImport.projectMapCsv
161+
162+
Optional name of CSV file (located in project root) containing multiple projects for migration. If set, it overrules `gitlab.projectId` and `github.repo`.
163+
CSV file must contain the following values: GitLab project ID, GitLab project path, GitHub project path, e.g. 213,gitlab_namespace/gitlab_projectname,github_namespace/github_projectname
164+
165+
### csvImport.gitlabProjectIdColumn
166+
167+
Column in CSV file containing GitLab project ID
168+
169+
### csvImport.gitlabProjectPathColumn
170+
171+
Column in CSV file containing GitLab path
172+
173+
### csvImport.githubProjectPathColumn
174+
175+
Column in CSV file containing GitHub path
176+
159177
### conversion
160178

161179
#### conversion.useLowerCaseLabels

‎src/githubHelper.ts‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -103,16 +103,17 @@ export class GithubHelper {
103103

104104
this.members = new Set<string>();
105105

106-
// TODO: won't work if ownerIsOrg is false
107-
githubApi.orgs.listMembers( {
108-
org: this.githubOwner,
109-
}).then(members => {
110-
for (let member of members.data) {
111-
this.members.add(member.login);
112-
}
113-
}).catch(err => {
114-
console.error(`Failed to fetch organization members: ${err}`);
115-
});
106+
if (this.githubOwnerIsOrg) {
107+
githubApi.orgs.listMembers( {
108+
org: this.githubOwner,
109+
}).then(members => {
110+
for (let member of members.data) {
111+
this.members.add(member.login);
112+
}
113+
}).catch(err => {
114+
console.error(`Failed to fetch organization members: ${err}`);
115+
});
116+
}
116117
}
117118

118119
/*

‎src/index.ts‎

Lines changed: 63 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
} from './githubHelper';
77
import { GitlabHelper, GitLabIssue, GitLabMilestone } from './gitlabHelper';
88
import settings from '../settings';
9+
import { readProjectsFromCsv } from './utils';
910

1011
import { Octokit as GitHubApi } from '@octokit/rest';
1112
import { throttling } from '@octokit/plugin-throttling';
@@ -94,17 +95,69 @@ const githubHelper = new GithubHelper(
9495
settings.useIssuesForAllMergeRequests
9596
);
9697

97-
// If no project id is given in settings.js, just return
98-
// all of the projects that this user is associated with.
99-
if (!settings.gitlab.projectId) {
100-
gitlabHelper.listProjects();
101-
} else {
102-
// user has chosen a project
103-
if (settings.github.recreateRepo === true) {
104-
recreate();
98+
let projectMap: Map<number, [string, string]> = new Map();
99+
100+
if (settings.csvImport?.projectMapCsv) {
101+
console.log(`Loading projects from CSV: ${settings.csvImport.projectMapCsv}`);
102+
projectMap = readProjectsFromCsv(
103+
settings.csvImport.projectMapCsv,
104+
settings.csvImport.gitlabProjectIdColumn,
105+
settings.csvImport.gitlabProjectPathColumn,
106+
settings.csvImport.githubProjectPathColumn
107+
);
108+
} else {
109+
projectMap.set(settings.gitlab.projectId, ['', '']);
105110
}
106-
migrate();
107-
}
111+
112+
(async () => {
113+
if (projectMap.size === 0 || (projectMap.size === 1 && projectMap.has(0))) {
114+
await gitlabHelper.listProjects();
115+
} else {
116+
for (const projectId of projectMap.keys()) {
117+
const paths = projectMap.get(projectId);
118+
119+
if (!paths) {
120+
console.warn(`Warning: No paths found for project ID ${projectId}, skipping`);
121+
continue;
122+
}
123+
124+
const [gitlabPath, githubPath] = paths;
125+
126+
console.log(`\n\n${'='.repeat(60)}`);
127+
if (gitlabPath) {
128+
console.log(`Processing Project ID: ${projectId} ${gitlabPath} → GitHub: ${githubPath}`);
129+
} else {
130+
console.log(`Processing Project ID: ${projectId}`);
131+
}
132+
console.log(`${'='.repeat(60)}\n`);
133+
134+
settings.gitlab.projectId = projectId;
135+
gitlabHelper.gitlabProjectId = projectId;
136+
137+
if (githubPath) {
138+
const githubParts = githubPath.split('/');
139+
if (githubParts.length === 2) {
140+
settings.github.owner = githubParts[0];
141+
settings.github.repo = githubParts[1];
142+
143+
githubHelper.githubOwner = githubParts[0];
144+
githubHelper.githubRepo = githubParts[1];
145+
} else {
146+
settings.github.repo = githubPath;
147+
githubHelper.githubRepo = githubPath;
148+
}
149+
}
150+
151+
if (settings.github.recreateRepo === true) {
152+
await recreate();
153+
}
154+
await migrate();
155+
}
156+
}
157+
})().catch(err => {
158+
console.error('Migration failed:', err);
159+
process.exit(1);
160+
});
108161

109162
// ----------------------------------------------------------------------------
110163

‎src/settings.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ export default interface Settings {
99
projectmap: {
1010
[key: string]: string;
1111
};
12+
csvImport?:{
13+
projectMapCsv?: string;
14+
gitlabProjectIdColumn?: number;
15+
gitlabProjectPathColumn?: number;
16+
githubProjectPathColumn?: number;
17+
}
1218
conversion: {
1319
useLowerCaseLabels: boolean;
1420
addIssueInformation: boolean;

‎src/utils.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,88 @@ import settings from '../settings';
33
import * as mime from 'mime-types';
44
import * as path from 'path';
55
import * as crypto from 'crypto';
6+
import * as fs from 'fs';
67
import S3 from 'aws-sdk/clients/s3';
78
import { GitlabHelper } from './gitlabHelper';
89

910
export const sleep = (milliseconds: number) => {
1011
return new Promise(resolve => setTimeout(resolve, milliseconds));
1112
};
1213

14+
export const readProjectsFromCsv = (
15+
filePath: string,
16+
idColumn: number = 0,
17+
gitlabPathColumn: number = 1,
18+
githubPathColumn: number = 2
19+
): Map<number, [string, string]> => {
20+
try {
21+
if (!fs.existsSync(filePath)) {
22+
throw new Error(`CSV file not found: ${filePath}`);
23+
}
24+
25+
const content = fs.readFileSync(filePath, 'utf-8');
26+
const lines = content.split(/\r?\n/);
27+
const projectMap = new Map<number, [string, string]>();
28+
let headerSkipped = false;
29+
30+
for (let i = 0; i < lines.length; i++) {
31+
const line = lines[i].trim();
32+
33+
if (!line || line.startsWith('#')) {
34+
continue;
35+
}
36+
37+
const values = line.split(',').map(v => v.trim());
38+
const maxColumn = Math.max(idColumn, gitlabPathColumn, githubPathColumn);
39+
40+
if (maxColumn >= values.length) {
41+
console.warn(`Warning: Line ${i + 1} has only ${values.length} column(s), skipping (need column ${maxColumn})`);
42+
if (!headerSkipped) {
43+
headerSkipped = true;
44+
}
45+
continue;
46+
}
47+
48+
const idStr = values[idColumn];
49+
const gitlabPath = values[gitlabPathColumn];
50+
const githubPath = values[githubPathColumn];
51+
52+
if (!headerSkipped) {
53+
const num = parseInt(idStr, 10);
54+
if (isNaN(num) || idStr.toLowerCase().includes('id') || idStr.toLowerCase().includes('project')) {
55+
console.log(`Skipping CSV header row: "${line}"`);
56+
headerSkipped = true;
57+
continue;
58+
}
59+
headerSkipped = true;
60+
}
61+
62+
if (!idStr || !gitlabPath || !githubPath) {
63+
console.warn(`Warning: Line ${i + 1} has empty values, skipping`);
64+
continue;
65+
}
66+
67+
const projectId = parseInt(idStr, 10);
68+
if (isNaN(projectId)) {
69+
console.warn(`Warning: Line ${i + 1}: Invalid project ID "${idStr}", skipping`);
70+
continue;
71+
}
72+
73+
projectMap.set(projectId, [gitlabPath, githubPath]);
74+
}
75+
76+
if (projectMap.size === 0) {
77+
throw new Error(`No valid project mappings found in CSV file: ${filePath}`);
78+
}
79+
80+
console.log(`✓ Loaded ${projectMap.size} project mappings from CSV`);
81+
return projectMap;
82+
} catch (err) {
83+
console.error(`Error reading project mapping CSV file: ${err.message}`);
84+
throw err;
85+
}
86+
};
87+
1388
// Creates new attachments and replaces old links
1489
export const migrateAttachments = async (
1590
body: string,

0 commit comments

Comments
 (0)