diff --git a/src/lib/server/db/migrations/005_update_published_at.sql b/src/lib/server/db/migrations/005_update_published_at.sql new file mode 100644 index 000000000..c9043165d --- /dev/null +++ b/src/lib/server/db/migrations/005_update_published_at.sql @@ -0,0 +1 @@ +update content set published_at = created_at; \ No newline at end of file diff --git a/src/lib/server/services/importers/github.ts b/src/lib/server/services/importers/github.ts index d883d89ba..400f8e8bb 100644 --- a/src/lib/server/services/importers/github.ts +++ b/src/lib/server/services/importers/github.ts @@ -71,60 +71,27 @@ export class GitHubImporter { } if (!response || !response.headers.get('content-type')?.includes('image')) { - throw new Error('Failed to fetch image') + const message = + 'Failed to download OG image. Rate limit exceeded: ' + contentData.metadata.ogImage + + console.error(message) + + throw new Error(message) } - const dir = path.join(STATE_DIRECTORY, 'files', 'gh', repository.full_name) + const dir = path.join(STATE_DIRECTORY, 'files', 'gh', owner, repo) const extension = response.headers.get('content-type')?.split('/').at(-1) const file_path = path.join(dir, 'thumbnail.' + extension) fs.mkdirSync(dir, { recursive: true }) fs.writeFileSync(file_path, Buffer.from(await response.arrayBuffer())) - const thumbnail = path.join('/files', 'gh', repository.full_name, 'thumbnail.' + extension) + const thumbnail = path.join('/files', 'gh', owner, repo, 'thumbnail.' + extension) contentData.metadata.thumbnail = thumbnail - return this.externalContentService.upsertExternalContent(contentData) - } - - /** - * Import multiple repositories from a user or organization - */ - async importUserRepositories( - username: string, - options?: { - maxRepos?: number - filterTopics?: string[] - minStars?: number - } - ): Promise { - const repositories = await this.fetchUserRepositories(username) - let filteredRepos = repositories - - // Apply filters - if (options?.filterTopics && options.filterTopics.length > 0) { - filteredRepos = filteredRepos.filter((repo) => - repo.topics.some((topic) => options.filterTopics!.includes(topic)) - ) - } - - if (options?.minStars) { - filteredRepos = filteredRepos.filter((repo) => repo.stargazers_count >= options.minStars!) - } - - if (options?.maxRepos) { - filteredRepos = filteredRepos.slice(0, options.maxRepos) - } - - const importedIds: string[] = [] - for (const repo of filteredRepos) { - const readme = await this.fetchReadme(repo.owner.login, repo.name) - const contentData = this.transformRepositoryToContent(repo, readme) - const id = await this.externalContentService.upsertExternalContent(contentData) - if (id) importedIds.push(id) - } + console.log('Successfully downloaded OG image: ' + contentData.metadata.ogImage) - return importedIds + return this.externalContentService.upsertExternalContent(contentData) } /** @@ -134,15 +101,15 @@ export class GitHubImporter { if (this.cacheService) { return this.cacheService.cachify({ key: `github:repo:${owner}:${repo}`, - getFreshValue: () => this._fetchRepositoryDirectly(owner, repo), + getFreshValue: () => this.fetchRepositoryDirectly(owner, repo), ttl: 60 * 60 * 1000, // 1 hour swr: 60 * 60 * 1000 * 24 // 24 hours }) } - return this._fetchRepositoryDirectly(owner, repo) + return this.fetchRepositoryDirectly(owner, repo) } - private async _fetchRepositoryDirectly( + private async fetchRepositoryDirectly( owner: string, repo: string ): Promise { @@ -170,57 +137,6 @@ export class GitHubImporter { } } - /** - * Fetch repositories for a user or organization - */ - private async fetchUserRepositories(username: string): Promise { - if (this.cacheService) { - return this.cacheService.cachify({ - key: `github:user:${username}:repos`, - getFreshValue: () => this._fetchUserRepositoriesDirectly(username), - ttl: 60 * 60 * 1000, // 1 hour - swr: 60 * 60 * 1000 * 24 // 24 hours - }) - } - return this._fetchUserRepositoriesDirectly(username) - } - - private async _fetchUserRepositoriesDirectly(username: string): Promise { - try { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3+json', - 'User-Agent': 'SvelteSociety-Importer' - } - - if (this.token) { - headers['Authorization'] = `Bearer ${this.token}` - } - - // Try organization endpoint first, fall back to user endpoint - let response = await fetch( - `${this.apiBaseUrl}/orgs/${username}/repos?per_page=100&sort=updated`, - { headers } - ) - - if (!response.ok) { - response = await fetch( - `${this.apiBaseUrl}/users/${username}/repos?per_page=100&sort=updated`, - { headers } - ) - } - - if (!response.ok) { - console.error(`Failed to fetch repositories for ${username}: ${response.statusText}`) - return [] - } - - return response.json() - } catch (error) { - console.error('Error fetching user repositories:', error) - return [] - } - } - /** * Fetch README content */ diff --git a/src/routes/files/[...path]/+server.ts b/src/routes/files/[...path]/+server.ts index 103e06247..d8d1e2f52 100644 --- a/src/routes/files/[...path]/+server.ts +++ b/src/routes/files/[...path]/+server.ts @@ -2,6 +2,8 @@ import fs from 'node:fs' import path from 'node:path' import { Readable } from 'node:stream' +import { GitHubImporter } from '$lib/server/services/importers/github' + const { STATE_DIRECTORY = '.state_directory' } = process.env if (!fs.existsSync(STATE_DIRECTORY)) { @@ -9,7 +11,7 @@ if (!fs.existsSync(STATE_DIRECTORY)) { } /** @type {import('./$types').RequestHandler} */ -export async function GET({ params, request }) { +export async function GET({ params, request, locals }) { const p = decodeURIComponent(params.path) const file_path = path.normalize(path.join(STATE_DIRECTORY, 'files', p)) @@ -31,27 +33,10 @@ export async function GET({ params, request }) { if (!exists || Date.now() - stats?.mtime.getTime() >= T) { const [owner, repo] = rest - const og_image_url = `https://opengraph.githubassets.com/${crypto.randomUUID()}/${owner}/${repo}` - - let response - - try { - response = await fetch(og_image_url) - } catch (error) { - console.error(error) - break - } - - if (!response.ok) { - console.log('Could not refresh OG image. Rate limit exceeded: ' + og_image_url) - break - } - - fs.writeFileSync(file_path, Buffer.from(await response.arrayBuffer())) + const importer = new GitHubImporter(locals.externalContentService, locals.cacheService) + const contentId = await importer.importRepository(owner, repo) stats = fs.statSync(file_path) - - console.log('Refreshed OG image: ' + og_image_url) } break