Skip to content

feat: autoupdate gh content #821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
karimfromjordan wants to merge 2 commits into
base: staging
Choose a base branch
from
Draft

feat: autoupdate gh content #821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b542c39
wip
karimfromjordan Oct 21, 2025
2acc4e1
wip
karimfromjordan Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/lib/server/db/migrations/005_update_published_at.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
update content set published_at = created_at;
110 changes: 13 additions & 97 deletions src/lib/server/services/importers/github.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,60 +71,27 @@ export class GitHubImporter {
}

if (!response || !response.headers.get('content-type')?.includes('image')) {
throw new Error('Failed to fetch image')
const message =
'Failed to download OG image. Rate limit exceeded: ' + contentData.metadata.ogImage

console.error(message)

throw new Error(message)
}

const dir = path.join(STATE_DIRECTORY, 'files', 'gh', repository.full_name)
const dir = path.join(STATE_DIRECTORY, 'files', 'gh', owner, repo)
const extension = response.headers.get('content-type')?.split('/').at(-1)
const file_path = path.join(dir, 'thumbnail.' + extension)

fs.mkdirSync(dir, { recursive: true })
fs.writeFileSync(file_path, Buffer.from(await response.arrayBuffer()))

const thumbnail = path.join('/files', 'gh', repository.full_name, 'thumbnail.' + extension)
const thumbnail = path.join('/files', 'gh', owner, repo, 'thumbnail.' + extension)
contentData.metadata.thumbnail = thumbnail

return this.externalContentService.upsertExternalContent(contentData)
}

/**
* Import multiple repositories from a user or organization
*/
async importUserRepositories(
username: string,
options?: {
maxRepos?: number
filterTopics?: string[]
minStars?: number
}
): Promise<string[]> {
const repositories = await this.fetchUserRepositories(username)
let filteredRepos = repositories

// Apply filters
if (options?.filterTopics && options.filterTopics.length > 0) {
filteredRepos = filteredRepos.filter((repo) =>
repo.topics.some((topic) => options.filterTopics!.includes(topic))
)
}

if (options?.minStars) {
filteredRepos = filteredRepos.filter((repo) => repo.stargazers_count >= options.minStars!)
}

if (options?.maxRepos) {
filteredRepos = filteredRepos.slice(0, options.maxRepos)
}

const importedIds: string[] = []
for (const repo of filteredRepos) {
const readme = await this.fetchReadme(repo.owner.login, repo.name)
const contentData = this.transformRepositoryToContent(repo, readme)
const id = await this.externalContentService.upsertExternalContent(contentData)
if (id) importedIds.push(id)
}
console.log('Successfully downloaded OG image: ' + contentData.metadata.ogImage)

return importedIds
return this.externalContentService.upsertExternalContent(contentData)
}

/**
Expand All @@ -134,15 +101,15 @@ export class GitHubImporter {
if (this.cacheService) {
return this.cacheService.cachify({
key: `github:repo:${owner}:${repo}`,
getFreshValue: () => this._fetchRepositoryDirectly(owner, repo),
getFreshValue: () => this.fetchRepositoryDirectly(owner, repo),
ttl: 60 * 60 * 1000, // 1 hour
swr: 60 * 60 * 1000 * 24 // 24 hours
})
}
return this._fetchRepositoryDirectly(owner, repo)
return this.fetchRepositoryDirectly(owner, repo)
}

private async _fetchRepositoryDirectly(
private async fetchRepositoryDirectly(
owner: string,
repo: string
): Promise<GitHubRepository | null> {
Expand Down Expand Up @@ -170,57 +137,6 @@ export class GitHubImporter {
}
}

/**
* Fetch repositories for a user or organization
*/
private async fetchUserRepositories(username: string): Promise<GitHubRepository[]> {
if (this.cacheService) {
return this.cacheService.cachify({
key: `github:user:${username}:repos`,
getFreshValue: () => this._fetchUserRepositoriesDirectly(username),
ttl: 60 * 60 * 1000, // 1 hour
swr: 60 * 60 * 1000 * 24 // 24 hours
})
}
return this._fetchUserRepositoriesDirectly(username)
}

private async _fetchUserRepositoriesDirectly(username: string): Promise<GitHubRepository[]> {
try {
const headers: HeadersInit = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': 'SvelteSociety-Importer'
}

if (this.token) {
headers['Authorization'] = `Bearer ${this.token}`
}

// Try organization endpoint first, fall back to user endpoint
let response = await fetch(
`${this.apiBaseUrl}/orgs/${username}/repos?per_page=100&sort=updated`,
{ headers }
)

if (!response.ok) {
response = await fetch(
`${this.apiBaseUrl}/users/${username}/repos?per_page=100&sort=updated`,
{ headers }
)
}

if (!response.ok) {
console.error(`Failed to fetch repositories for ${username}: ${response.statusText}`)
return []
}

return response.json()
} catch (error) {
console.error('Error fetching user repositories:', error)
return []
}
}

/**
* Fetch README content
*/
Expand Down
25 changes: 5 additions & 20 deletions src/routes/files/[...path]/+server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@ import fs from 'node:fs'
import path from 'node:path'
import { Readable } from 'node:stream'

import { GitHubImporter } from '$lib/server/services/importers/github'

const { STATE_DIRECTORY = '.state_directory' } = process.env

if (!fs.existsSync(STATE_DIRECTORY)) {
fs.mkdirSync(STATE_DIRECTORY, { recursive: true })
}

/** @type {import('./$types').RequestHandler} */
export async function GET({ params, request }) {
export async function GET({ params, request, locals }) {
const p = decodeURIComponent(params.path)

const file_path = path.normalize(path.join(STATE_DIRECTORY, 'files', p))
Expand All @@ -31,27 +33,10 @@ export async function GET({ params, request }) {
if (!exists || Date.now() - stats?.mtime.getTime() >= T) {
const [owner, repo] = rest

const og_image_url = `https://opengraph.githubassets.com/${crypto.randomUUID()}/${owner}/${repo}`

let response

try {
response = await fetch(og_image_url)
} catch (error) {
console.error(error)
break
}

if (!response.ok) {
console.log('Could not refresh OG image. Rate limit exceeded: ' + og_image_url)
break
}

fs.writeFileSync(file_path, Buffer.from(await response.arrayBuffer()))
const importer = new GitHubImporter(locals.externalContentService, locals.cacheService)
const contentId = await importer.importRepository(owner, repo)

stats = fs.statSync(file_path)

console.log('Refreshed OG image: ' + og_image_url)
}

break
Expand Down