diff --git a/.dev.vars.example b/.dev.vars.example index 75a1c71..b25da77 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -1,3 +1 @@ -SUPABASE_URL= -SUPABASE_ANON_KEY= REPLICATE_API_TOKEN= diff --git a/.github/workflows/update_repos.yml b/.github/workflows/update_repos.yml index 4d1241e..2b59ac5 100644 --- a/.github/workflows/update_repos.yml +++ b/.github/workflows/update_repos.yml @@ -23,7 +23,3 @@ jobs: - name: Call updateContent script run: npm run updateContent - env: - SUPABASE_URL: ${{ secrets.SUPABASE_URL }} - SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} - REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }} diff --git a/README.md b/README.md index 33665bd..6fffff7 100644 --- a/README.md +++ b/README.md @@ -2,26 +2,44 @@ It's really hard to keep up with open source machine learning. Almost every new Python repo on GitHub is an ML repo, so we made a website that displays all the latest Python repos in a HN-like list. We also added Replicate and HuggingFace models, and posts from r/{LocalLLaMA,MachineLearning,StableDiffusion}. -The website is updated every hour. +The website is updated every hour via GitHub Actions. ## Local Dev ```bash npm install -cp .dev.vars.example .dev.vars # fill in values -npm run dev +cp .dev.vars.example .dev.vars # add your REPLICATE_API_TOKEN +npm run dev # uses local D1 database ``` -## Deploy +To develop against the remote D1 database: +```bash +wrangler dev --remote +``` + +## Deploy (New Setup) ```bash -wrangler secret put SUPABASE_URL -wrangler secret put SUPABASE_ANON_KEY +# 1. Create D1 database +wrangler d1 create hype + +# 2. Update database_id in wrangler.jsonc with the ID from step 1 + +# 3. Run migration +wrangler d1 execute hype --remote --file=migrations/0001_init.sql + +# 4. Set secrets wrangler secret put REPLICATE_API_TOKEN + +# 5. Deploy npm run deploy ``` -Content updates hourly. Manual trigger: `npm run updateContent`. +## Data Updates + +Content updates hourly via GitHub Actions, which calls the `/api/update` endpoint. + +Manual trigger: `npm run updateContent` ## Want to run AI models yourself? diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..b405e5d --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,19 @@ +-- Create repositories table +CREATE TABLE IF NOT EXISTS repositories ( + id TEXT NOT NULL, + source TEXT NOT NULL, + username TEXT NOT NULL, + name TEXT, + description TEXT, + stars INTEGER DEFAULT 0, + url TEXT, + created_at TEXT, + inserted_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (id, source) +); + +-- Indexes for common queries +CREATE INDEX IF NOT EXISTS idx_repos_source ON repositories(source); +CREATE INDEX IF NOT EXISTS idx_repos_created ON repositories(created_at); +CREATE INDEX IF NOT EXISTS idx_repos_inserted ON repositories(inserted_at); +CREATE INDEX IF NOT EXISTS idx_repos_stars ON repositories(stars DESC); diff --git a/package-lock.json b/package-lock.json index b26ff2a..93ffb6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,6 @@ "": { "name": "hype", "dependencies": { - "@supabase/supabase-js": "^2.12.1", "chanfana": "2.8.3", "hono": "4.11.1", "mustache": "^4.2.0", @@ -1901,86 +1900,6 @@ "dev": true, "license": "CC0-1.0" }, - "node_modules/@supabase/auth-js": { - "version": "2.89.0", - "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.89.0.tgz", - "integrity": "sha512-wiWZdz8WMad8LQdJMWYDZ2SJtZP5MwMqzQq3ehtW2ngiI3UTgbKiFrvMUUS3KADiVlk4LiGfODB2mrYx7w2f8w==", - "license": "MIT", - "dependencies": { - "tslib": "2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@supabase/functions-js": { - "version": "2.89.0", - "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.89.0.tgz", - "integrity": "sha512-XEueaC5gMe5NufNYfBh9kPwJlP5M2f+Ogr8rvhmRDAZNHgY6mI35RCkYDijd92pMcNM7g8pUUJov93UGUnqfyw==", - "license": "MIT", - "dependencies": { - "tslib": "2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@supabase/postgrest-js": { - "version": "2.89.0", - "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-2.89.0.tgz", - "integrity": "sha512-/b0fKrxV9i7RNOEXMno/I1862RsYhuUo+Q6m6z3ar1f4ulTMXnDfv0y4YYxK2POcgrOXQOgKYQx1eArybyNvtg==", - "license": "MIT", - "dependencies": { - "tslib": "2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@supabase/realtime-js": { - "version": "2.89.0", - "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.89.0.tgz", - "integrity": "sha512-aMOvfDb2a52u6PX6jrrjvACHXGV3zsOlWRzZsTIOAJa0hOVvRp01AwC1+nLTGUzxzezejrYeCX+KnnM1xHdl+w==", - "license": "MIT", - "dependencies": { - "@types/phoenix": "^1.6.6", - "@types/ws": "^8.18.1", - "tslib": "2.8.1", - "ws": "^8.18.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@supabase/storage-js": { - "version": "2.89.0", - "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.89.0.tgz", - "integrity": "sha512-6zKcXofk/M/4Eato7iqpRh+B+vnxeiTumCIP+Tz26xEqIiywzD9JxHq+udRrDuv6hXE+pmetvJd8n5wcf4MFRQ==", - "license": "MIT", - "dependencies": { - "iceberg-js": "^0.8.1", - "tslib": "2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@supabase/supabase-js": { - "version": "2.89.0", - "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.89.0.tgz", - "integrity": "sha512-KlaRwSfFA0fD73PYVMHj5/iXFtQGCcX7PSx0FdQwYEEw9b2wqM7GxadY+5YwcmuEhalmjFB/YvqaoNVF+sWUlg==", - "license": "MIT", - "dependencies": { - "@supabase/auth-js": "2.89.0", - "@supabase/functions-js": "2.89.0", - "@supabase/postgrest-js": "2.89.0", - "@supabase/realtime-js": "2.89.0", - "@supabase/storage-js": "2.89.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2017,26 +1936,12 @@ "version": "24.10.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.16.0" } }, - "node_modules/@types/phoenix": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.7.tgz", - "integrity": "sha512-oN9ive//QSBkf19rfDv45M7eZPi0eEXylht2OLEXicu5b4KoQ1OzXIw+xDSGWxSxe1JmepRR/ZH283vsu518/Q==", - "license": "MIT" - }, - "node_modules/@types/ws": { - "version": "8.18.1", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", - "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@vitest/expect": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", @@ -2100,7 +2005,6 @@ "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/utils": "3.2.4", "pathe": "^2.0.3", @@ -2116,7 +2020,6 @@ "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/pretty-format": "3.2.4", "magic-string": "^0.30.17", @@ -2515,15 +2418,6 @@ "node": ">=16.9.0" } }, - "node_modules/iceberg-js": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/iceberg-js/-/iceberg-js-0.8.1.tgz", - "integrity": "sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==", - "license": "MIT", - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/is-arrayish": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", @@ -2730,7 +2624,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -3017,7 +2910,9 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "dev": true, + "license": "0BSD", + "optional": true }, "node_modules/typescript": { "version": "5.9.3", @@ -3047,6 +2942,7 @@ "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, "license": "MIT" }, "node_modules/unenv": { @@ -3055,7 +2951,6 @@ "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "pathe": "^2.0.3" } @@ -3066,7 +2961,6 @@ "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -3165,7 +3059,6 @@ "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.4", @@ -3257,7 +3150,6 @@ "dev": true, "hasInstallScript": true, "license": "Apache-2.0", - "peer": true, "bin": { "workerd": "bin/workerd" }, @@ -3927,7 +3819,6 @@ "dev": true, "hasInstallScript": true, "license": "Apache-2.0", - "peer": true, "bin": { "workerd": "bin/workerd" }, @@ -3974,27 +3865,6 @@ "url": "https://github.com/sponsors/colinhacks" } }, - "node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/yaml": { "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", @@ -4049,7 +3919,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.67.tgz", "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 5094cea..08e2d13 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "private": true, "type": "module", "dependencies": { - "@supabase/supabase-js": "^2.12.1", "chanfana": "2.8.3", "hono": "4.11.1", "mustache": "^4.2.0", diff --git a/src/db.ts b/src/db.ts index b59481d..22684a0 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,12 +1,7 @@ -import { createClient, SupabaseClient } from "@supabase/supabase-js"; import type { Post } from "./types"; const BANNED_STRINGS = ["nft", "crypto", "telegram", "clicker", "solana", "stealer"]; -function getClient(env: Env): SupabaseClient { - return createClient(env.SUPABASE_URL, env.SUPABASE_ANON_KEY); -} - function isValidPost(post: Post): boolean { const name = post.name?.toLowerCase() || ""; const desc = post.description?.toLowerCase() || ""; @@ -37,47 +32,58 @@ function getFromDate(filter: FilterType): Date { export const posts = { async query(env: Env, options: { filter: FilterType; sources: string[] }): Promise { - const client = getClient(env); const fromDate = getFromDate(options.filter); const sourcesLower = options.sources.map((s) => s.toLowerCase()); - const { data, error } = await client - .from("repositories") - .select("*") - .order("stars", { ascending: false }) - .limit(500) - .in("source", sourcesLower) - .gt("created_at", fromDate.toISOString()) - .gt("inserted_at", fromDate.toISOString()); + const placeholders = sourcesLower.map(() => "?").join(", "); + const query = ` + SELECT * FROM repositories + WHERE source IN (${placeholders}) + AND created_at > ? + AND inserted_at > ? + ORDER BY stars DESC + LIMIT 500 + `; - if (error) throw new Error(`Database error: ${error.message}`); + const { results } = await env.DB.prepare(query) + .bind(...sourcesLower, fromDate.toISOString(), fromDate.toISOString()) + .all(); - const filtered = (data || []).filter(isValidPost); + const filtered = (results || []).filter(isValidPost); filtered.sort((a, b) => scorePost(b) - scorePost(a)); return filtered; }, async upsert(env: Env, post: Post): Promise { - const client = getClient(env); - const { error } = await client.from("repositories").upsert( - { - id: post.id, - source: post.source, - username: post.username, - name: post.name, - description: post.description, - stars: post.stars, - url: post.url, - created_at: post.created_at, - }, - { onConflict: "id,source" } - ); - if (error) throw new Error(`Database error upserting ${post.id}: ${error.message}`); + const query = ` + INSERT INTO repositories (id, source, username, name, description, stars, url, created_at, inserted_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now')) + ON CONFLICT (id, source) DO UPDATE SET + username = excluded.username, + name = excluded.name, + description = excluded.description, + stars = excluded.stars, + url = excluded.url, + created_at = excluded.created_at + `; + + await env.DB.prepare(query) + .bind( + post.id, + post.source, + post.username, + post.name, + post.description, + post.stars, + post.url, + post.created_at + ) + .run(); }, async getLastUpdated(env: Env): Promise { - const client = getClient(env); - const { data } = await client.rpc("repositories_last_modified"); - return data || null; + const query = `SELECT MAX(inserted_at) as last_updated FROM repositories`; + const { results } = await env.DB.prepare(query).all<{ last_updated: string | null }>(); + return results?.[0]?.last_updated || null; }, }; diff --git a/src/templates/page.html b/src/templates/page.html index f6dc6a6..f178dcb 100644 --- a/src/templates/page.html +++ b/src/templates/page.html @@ -11,50 +11,49 @@
-
- Hype - What is this? +
+ Hype + What is this?
{{#filterLinks}} - {{^first}}|{{/first}} - {{label}} + {{^first}}|{{/first}} + {{label}} {{/filterLinks}}
-
-
+
+
{{#sources}} {{/sources}}
- Last updated + Updated
    {{#posts}} -
  • - {{index}}. -
    -
    - {{displayName}} - {{icon}} - {{stars}} +
  • + {{index}}. +
    +
    + {{displayName}} + {{icon}} {{stars}}
    -

    {{description}}

    +

    {{description}}

    • {{/posts}}
-