From 0f64a8177c7d4a7ed93044c06c773ae4eb748d79 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Tue, 23 May 2023 01:26:00 +0100 Subject: [PATCH 01/36] Add stub `examples/nextjs-import-airbyte-github-export-seafowl/` --- .../.gitignore | 1 + .../README.md | 26 +++++++ .../next-env.d.ts | 5 ++ .../package.json | 20 ++++++ .../pages/index.tsx | 67 +++++++++++++++++++ .../tsconfig.json | 20 ++++++ examples/yarn.lock | 17 ++++- 7 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/.gitignore create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/README.md create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/package.json create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore new file mode 100644 index 0000000..a680367 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore @@ -0,0 +1 @@ +.next diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/README.md b/examples/nextjs-import-airbyte-github-export-seafowl/README.md new file mode 100644 index 0000000..97e13da --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/README.md @@ -0,0 +1,26 @@ +# End-to-End Example: Use `airbyte-github` to import GitHub repository into Splitgraph, then export it to Seafowl, via Next.js API routes + +This is a full end-to-end example demonstrating importing data to Splitgraph +(using the `airbyte-github` plugin), exporting it to Seafowl (using the +`export-to-seafowl` plugin), and then querying it (with `DbSeafowl` and React +hooks from `@madatdata/react`). The importers and exporting of data is triggered +by backend API routes (e.g. the Vecel runtime), which execute in an environment +with secrets (an `API_SECRET` for Splitgraph, and a GitHub access token for +`airbyte-github`). The client side queries Seafowl directly by sending raw SQL +queries in HTP requests, which is what Seafowl is ultimately designed for. + +## Try Now + +### Preview Immediately + +_No signup required, just click the button!_ + +[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/splitgraph/madatdata/tree/main/examples/nextjs-import-airbyte-github-export-seafowl?file=pages/index.tsx) + +[![Open with CodeSandbox](https://assets.codesandbox.io/github/button-edit-lime.svg)](https://codesandbox.io/p/sandbox/github/splitgraph/madatdata/main/examples/nextjs-import-airbyte-github-export-seafowl?file=pages/index.tsx&hardReloadOnChange=true&startScript=dev&node=16&port=3000) + +### Or, deploy to Vercel (signup required) + +_Signup, fork the repo, and import it_ + +[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/git/external?repository-url=https://github.com/splitgraph/madatdata/tree/main/examples/nextjs-import-airbyte-github-export-seafowl&project-name=madatdata-basic-hooks&repository-name=madatdata-nextjs-basic-hooks) diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts new file mode 100644 index 0000000..4f11a03 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/package.json b/examples/nextjs-import-airbyte-github-export-seafowl/package.json new file mode 100644 index 0000000..6d17d38 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/package.json @@ -0,0 +1,20 @@ +{ + "private": true, + "scripts": { + "dev": "yarn next", + "build": "yarn next build", + "start": "yarn next start" + }, + "dependencies": { + "@madatdata/core": "latest", + "@madatdata/react": "latest", + "next": "latest", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "^18.0.0", + "@types/react": "^18.0.14", + "typescript": "^4.7.4" + } +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx new file mode 100644 index 0000000..c3a646d --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx @@ -0,0 +1,67 @@ +import { + SqlProvider, + useSql, + makeSplitgraphHTTPContext, +} from "@madatdata/react"; +import { useMemo } from "react"; + +const ExampleComponentUsingSQL = () => { + const { loading, error, response } = useSql<{ + origin_airport: string; + destination_airport: string; + origin_city: string; + destination_city: string; + passengers: number; + seats: number; + flights: number; + distance: number; + fly_month: string; + origin_pop: number; + destination_pop: number; + id: number; + }>( + `SELECT + "origin_airport", + "destination_airport", + "origin_city", + "destination_city", + "passengers", + "seats", + "flights", + "distance", + "fly_month", + "origin_pop", + "destination_pop", + "id" +FROM + "splitgraph/domestic_us_flights:latest"."flights" +LIMIT 100;` + ); + + return ( + 
+      {JSON.stringify({ loading, error, response }, null, 2)}
+
+ ); +}; + +const SplitgraphSampleQuery = () => { + const splitgraphDataContext = useMemo( + () => makeSplitgraphHTTPContext({ credential: null }), + [] + ); + + // Uses splitgraph.com by default (anon access supported for public data) + return ( + + + + ); +}; + +export default SplitgraphSampleQuery; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json new file mode 100644 index 0000000..16bb209 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "es5", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": false, + "forceConsistentCasingInFileNames": true, + "noEmit": true, + "incremental": true, + "esModuleInterop": true, + "module": "esnext", + "resolveJsonModule": true, + "moduleResolution": "Node", + "isolatedModules": true, + "jsx": "preserve" + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] +} diff --git a/examples/yarn.lock b/examples/yarn.lock index dfc7d9a..3cbd334 100644 --- a/examples/yarn.lock +++ b/examples/yarn.lock @@ -533,7 +533,7 @@ __metadata: languageName: node linkType: hard -"@madatdata/core@npm:0.0.11": +"@madatdata/core@npm:0.0.11, @madatdata/core@npm:latest": version: 0.0.11 resolution: "@madatdata/core@npm:0.0.11" dependencies: @@ -1894,6 +1894,21 @@ __metadata: languageName: node linkType: hard +"nextjs-import-airbyte-github-export-seafowl-acabed@workspace:nextjs-import-airbyte-github-export-seafowl": + version: 0.0.0-use.local + resolution: "nextjs-import-airbyte-github-export-seafowl-acabed@workspace:nextjs-import-airbyte-github-export-seafowl" + dependencies: + "@madatdata/core": latest + "@madatdata/react": latest + "@types/node": ^18.0.0 + "@types/react": ^18.0.14 + next: latest + react: 18.2.0 + react-dom: 18.2.0 + typescript: ^4.7.4 + languageName: unknown + linkType: soft + "node-fetch@npm:2.6.7": version: 2.6.7 resolution: "node-fetch@npm:2.6.7" From 776e11f77a486dee5d07fa0217aa8c6da701340c Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Tue, 23 May 2023 20:13:23 +0100 Subject: [PATCH 02/36] Stub out layout and sidebar of GitHub analytics example Assisted by the one and only GPT-4 --- .../components/BaseLayout.module.css | 37 ++++++++++ .../components/BaseLayout.tsx | 21 ++++++ .../components/Header.module.css | 44 ++++++++++++ .../components/Header.tsx | 25 +++++++ .../components/Logo.tsx | 46 ++++++++++++ .../components/Sidebar.module.css | 54 ++++++++++++++ .../components/Sidebar.tsx | 33 +++++++++ .../components/global-styles/reset.css | 71 +++++++++++++++++++ .../components/global-styles/theme.css | 14 ++++ .../pages/_app.tsx | 7 ++ .../pages/index.tsx | 36 +++++++++- 11 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css new file mode 100644 index 0000000..5cee596 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css @@ -0,0 +1,37 @@ +.container { + display: flex; + flex-direction: column; + height: 100vh; + background-color: var(--background); +} + +.header { + width: 100%; + position: sticky; + top: 0; + z-index: 100; + background-color: var(--header); + color: var(--text); +} + +.main { + display: flex; + flex-grow: 1; + overflow: hidden; +} + +.sidebar { + width: 20%; /* adjust as per your needs */ + overflow-y: auto; + /* add additional styles for your sidebar */ + color: var(--text); +} + +.content { + width: 80%; /* adjust as per your needs */ + overflow-y: auto; + position: relative; + /* add additional styles for your content area */ + color: var(--text); + background-color: var(--background); +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx new file mode 100644 index 0000000..c996b35 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx @@ -0,0 +1,21 @@ +import styles from "./BaseLayout.module.css"; +import { Header } from "./Header"; + +export const BaseLayout = ({ + children, + sidebar, +}: React.PropsWithChildren<{ + sidebar: React.ReactNode; +}>) => { + return ( +
+
+
+
+
+
{sidebar}
+
{children}
+
+
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css new file mode 100644 index 0000000..1b7d443 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.module.css @@ -0,0 +1,44 @@ +.header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 8px; +} + +.logo a { + text-decoration: none; +} + +.logo a:hover .wordmark { + text-shadow: 0 0 5px rgba(0, 0, 0, 0.1); +} + +.logo .wordmark { + color: var(--primary); + font-size: large; +} + +.logo .logo_link { + display: inline-flex; + align-items: center; + justify-content: center; +} + +.logo svg { + height: 36px; + margin: 8px; +} + +.nav { + margin: 16px; +} + +.nav a { + margin-left: 20px; + color: var(--secondary); + text-decoration: none; +} + +.nav a:hover { + text-decoration: underline; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx new file mode 100644 index 0000000..71de3fa --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Header.tsx @@ -0,0 +1,25 @@ +import React from "react"; +import Link from "next/link"; +import styles from "./Header.module.css"; +import { LogoSVG } from "./Logo"; + +interface HeaderProps {} + +export const Header: React.FC = () => { + return ( +
+
+ + +
GitHub Analytics
+ +
+ +
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx new file mode 100644 index 0000000..a0e82b8 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Logo.tsx @@ -0,0 +1,46 @@ +export const LogoSVG = ({ size }: { size: number }) => ( + + + + + + + + + + + + +); diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css new file mode 100644 index 0000000..a39db81 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.module.css @@ -0,0 +1,54 @@ +.sidebar { + background-color: var(--background); + border-right: 1px solid var(--header); + position: relative; +} + +.importButtonContainer { + position: sticky; + top: 0; + width: 100%; + background-color: var(--background); + display: flex; + align-items: center; + padding: 8px; + border-bottom: 1px dotted var(--sidebar); +} + +.importButton { + color: var(--background); + background-color: var(--secondary); + padding: 16px; + border-radius: 16px; +} + +.importButton { + text-decoration: none; + font-weight: bold; +} + +.importButton:hover { + text-shadow: 0 0 5px rgba(43, 0, 255, 0.5); +} + +.repoList { + list-style: none; + padding: 0; +} + +.repoList li { + margin-left: 0; + border-bottom: 1px dotted var(--sidebar); + padding-top: 8px; + padding-bottom: 8px; + padding-left: 16px; + padding-right: 16px; +} + +.repoList li a { + text-decoration: none; +} + +.repoList li a:hover { + text-decoration: underline; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx new file mode 100644 index 0000000..f3ccb2f --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx @@ -0,0 +1,33 @@ +import React from "react"; +import Link from "next/link"; +import styles from "./Sidebar.module.css"; + +export interface GitHubRepository { + namespace: string; + repository: string; +} + +interface SidebarProps { + repositories: GitHubRepository[]; +} + +export const Sidebar = ({ repositories }: React.PropsWithRef) => { + return ( + + ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css new file mode 100644 index 0000000..dd0e1d8 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/reset.css @@ -0,0 +1,71 @@ +/* https://www.joshwcomeau.com/css/custom-css-reset/ */ + +/* + 1. Use a more-intuitive box-sizing model. +*/ +*, +*::before, +*::after { + box-sizing: border-box; +} +/* + 2. Remove default margin +*/ +* { + margin: 0; +} +/* + 3. Allow percentage-based heights in the application +*/ +html, +body { + height: 100%; +} +/* + Typographic tweaks! + 4. Add accessible line-height + 5. Improve text rendering +*/ +body { + line-height: 1.5; + -webkit-font-smoothing: antialiased; +} +/* + 6. Improve media defaults +*/ +img, +picture, +video, +canvas, +svg { + display: block; + max-width: 100%; +} +/* + 7. Remove built-in form typography styles +*/ +input, +button, +textarea, +select { + font: inherit; +} +/* + 8. Avoid text overflows +*/ +p, +h1, +h2, +h3, +h4, +h5, +h6 { + overflow-wrap: break-word; +} +/* + 9. Create a root stacking context +*/ +#root, +#__next { + isolation: isolate; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css new file mode 100644 index 0000000..ecf2ea9 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css @@ -0,0 +1,14 @@ +:root { + --primary: #007bff; + --secondary: #ef00a7; + --background: #ffffff; + --header: #f2f6ff; + /* --header: #718096; */ + --sidebar: #718096; + --text: #1a202c; + --subtext: #718096; +} + +body { + font-family: Consolas, Monaco, "Andale Mono", "Ubuntu Mono", monospace; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx new file mode 100644 index 0000000..3a6568a --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/_app.tsx @@ -0,0 +1,7 @@ +import type { AppProps } from "next/app"; +import "../components/global-styles/reset.css"; +import "../components/global-styles/theme.css"; + +export default function GitHubAnalyticsApp({ Component, pageProps }: AppProps) { + return ; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx index c3a646d..bb79aff 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx @@ -5,6 +5,10 @@ import { } from "@madatdata/react"; import { useMemo } from "react"; +import { BaseLayout } from "../components/BaseLayout"; + +import { Sidebar, type GitHubRepository } from "../components/Sidebar"; + const ExampleComponentUsingSQL = () => { const { loading, error, response } = useSql<{ origin_airport: string; @@ -59,9 +63,39 @@ const SplitgraphSampleQuery = () => { // Uses splitgraph.com by default (anon access supported for public data) return ( - + }> + + ); }; export default SplitgraphSampleQuery; + +const sampleRepositories: GitHubRepository[] = [ + { namespace: "OpenTech", repository: "data-structures" }, + { namespace: "AiSolutions", repository: "machine-learning-api" }, + { namespace: "DevToolsInc", repository: "react-components" }, + { namespace: "QuantumComputing", repository: "quantum-algorithms" }, + { namespace: "GlobalNetworks", repository: "network-optimization" }, + { namespace: "CyberSec", repository: "firewall-config" }, + { namespace: "DataSci", repository: "data-analysis" }, + { namespace: "WebDevCo", repository: "responsive-templates" }, + { namespace: "CloudNet", repository: "cloud-infrastructure" }, + { namespace: "AiData", repository: "neural-networks" }, + { namespace: "DistributedSys", repository: "microservices-arch" }, + { namespace: "KernelDev", repository: "os-development" }, + { namespace: "FrontEndMagic", repository: "vue-utilities" }, + { namespace: "BackEndLogix", repository: "nodejs-server" }, + { namespace: "Securitech", repository: "encryption-utils" }, + { namespace: "FullStack", repository: "end-to-end-app" }, + { namespace: "DBMasters", repository: "database-design" }, + { namespace: "MobileApps", repository: "android-development" }, + { namespace: "GameFactory", repository: "game-engine" }, + { namespace: "WebAssembly", repository: "wasm-runtime" }, + { namespace: "RoboLogic", repository: "robot-navigation" }, + { namespace: "IoTDesign", repository: "iot-devices" }, + { namespace: "BlockchainTech", repository: "blockchain-network" }, + { namespace: "CryptoCoins", repository: "cryptocurrency" }, + { namespace: "VRWorld", repository: "vr-applications" }, +]; From 1f3b6d1c6d9159d84db9894d91100e8b7c93b786 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Wed, 24 May 2023 20:29:15 +0100 Subject: [PATCH 03/36] Add backend config and API routes for starting, awaiting import task --- .../.env.test.local | 18 +++ .../.gitignore | 3 + .../env-vars.d.ts | 41 ++++++ .../lib-backend/splitgraph-db.ts | 52 ++++++++ .../next.config.js | 33 +++++ .../package.json | 1 + .../pages/api/await-import-from-github.ts | 89 +++++++++++++ .../pages/api/start-import-from-github.ts | 117 ++++++++++++++++++ .../tsconfig.json | 2 +- 9 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/next.config.js create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local new file mode 100644 index 0000000..473bf7d --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local @@ -0,0 +1,18 @@ +# IMPORTANT: Put your own values in `.env.local` (a git-ignored file) when running this locally +# Configure them in Vercel settings when running in production +# This file is mostly to show which variables exist, since it's the only one checked into the repo. +# SEE: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables + +# Create your own API key and secret: https://www.splitgraph.com/connect +SPLITGRAPH_API_KEY="********************************" +SPLITGRAPH_API_SECRET="********************************" + +# Create a GitHub token that can query the repositories you want to connect +# For example, a token with read-only access to public repos is sufficient +# CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new +GITHUB_PAT_SECRET="github_pat_**********************_***********************************************************" + +# OPTIONAL: Set this environment variable to a proxy address to capture requests from API routes +# e.g. To intercept requests to Splitgraph API sent from madatdata libraries in API routes +# You can also set this by running: yarn dev-mitm (see package.json) +# MITMPROXY_ADDRESS="http://localhost:7979" diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore index a680367..d8dcd4f 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore +++ b/examples/nextjs-import-airbyte-github-export-seafowl/.gitignore @@ -1 +1,4 @@ .next +.env.local +.env.*.local +!.env.test.local diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts new file mode 100644 index 0000000..5328009 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts @@ -0,0 +1,41 @@ +namespace NodeJS { + interface ProcessEnv { + /** + * The API key of an existing Splitgraph account. + * + * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings. + * + * Get credentials: https://www.splitgraph.com/connect + */ + SPLITGRAPH_API_KEY: string; + + /** + * The API secret of an existing Splitgraph account. + * + * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings. + * + * Get credentials: https://www.splitgraph.com/connect + */ + SPLITGRAPH_API_SECRET: string; + + /** + * A GitHub personal access token that can be used for importing repositories. + * It will be passed to the Airbyte connector that runs on Splitgraph servers + * and ingests data from GitHub into Splitgraph. + * + * This should be defined in `.env.local` (a git-ignored file) or in Vercel settings. + * + * Create one here: https://github.com/settings/personal-access-tokens/new + */ + GITHUB_PAT_SECRET: string; + + /** + * Optional environment variable containing the address of a proxy instance + * through which to forward requests from API routes. See next.config.js + * for where it's setup. + * + * This is useful for debugging and development. + */ + MITMPROXY_ADDRESS?: string; + } +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts new file mode 100644 index 0000000..66ba751 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts @@ -0,0 +1,52 @@ +import { makeSplitgraphDb } from "@madatdata/core"; + +// TODO: fix plugin exports +import { makeDefaultPluginList } from "@madatdata/db-splitgraph"; +import { defaultSplitgraphHost } from "@madatdata/core"; + +const SPLITGRAPH_API_KEY = process.env.SPLITGRAPH_API_KEY; +const SPLITGRAPH_API_SECRET = process.env.SPLITGRAPH_API_SECRET; + +if (!SPLITGRAPH_API_KEY || !SPLITGRAPH_API_SECRET) { + throw new Error( + "Environment variable SPLITGRAPH_API_KEY or SPLITGRAPH_API_SECRET is not set." + + " See env-vars.d.ts for instructions." + ); +} + +const authenticatedCredential: Parameters< + typeof makeSplitgraphDb +>[0]["authenticatedCredential"] = { + apiKey: SPLITGRAPH_API_KEY, + apiSecret: SPLITGRAPH_API_SECRET, + anonymous: false, +}; + +// TODO: The access token can expire and silently fail? + +export const makeAuthenticatedSplitgraphDb = () => + makeSplitgraphDb({ + authenticatedCredential, + plugins: makeDefaultPluginList({ + graphqlEndpoint: defaultSplitgraphHost.baseUrls.gql, + authenticatedCredential, + }), + }); + +// TODO: export this utility function from the library +export const claimsFromJWT = (jwt?: string) => { + if (!jwt) { + return {}; + } + + const [_header, claims, _signature] = jwt + .split(".") + .map(fromBase64) + .slice(0, -1) // Signature is not parseable JSON + .map((o) => JSON.parse(o)); + + return claims; +}; + +const fromBase64 = (input: string) => + !!globalThis.Buffer ? Buffer.from(input, "base64").toString() : atob(input); diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/next.config.js b/examples/nextjs-import-airbyte-github-export-seafowl/next.config.js new file mode 100644 index 0000000..ac7ae64 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/next.config.js @@ -0,0 +1,33 @@ +const { ProxyAgent, setGlobalDispatcher } = require("undici"); + +// If running `yarn dev-mitm`, then setup the proxy with MITMPROXY_ADDRESS +// NOTE(FIXME): not all madatdata requests get sent through here for some reason +const setupProxy = () => { + if (!process.env.MITMPROXY_ADDRESS) { + return; + } + + const MITM = process.env.MITMPROXY_ADDRESS; + + console.log("MITM SETUP:", MITM); + + if (!process.env.GLOBAL_AGENT_HTTP_PROXY) { + process.env["GLOBAL_AGENT_HTTP_PROXY"] = MITM; + } + + process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0"; + + const mitmProxyOpts = { + uri: MITM, + connect: { + rejectUnauthorized: false, + requestCert: false, + }, + }; + + setGlobalDispatcher(new ProxyAgent(mitmProxyOpts)); +}; + +setupProxy(); + +module.exports = {}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/package.json b/examples/nextjs-import-airbyte-github-export-seafowl/package.json index 6d17d38..ef04766 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/package.json +++ b/examples/nextjs-import-airbyte-github-export-seafowl/package.json @@ -2,6 +2,7 @@ "private": true, "scripts": { "dev": "yarn next", + "dev-mitm": "MITMPROXY_ADDRESS=http://localhost:7979 yarn next", "build": "yarn next build", "start": "yarn next start" }, diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts new file mode 100644 index 0000000..d06aac9 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts @@ -0,0 +1,89 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db"; +import type { DeferredSplitgraphImportTask } from "@madatdata/db-splitgraph/plugins/importers/splitgraph-base-import-plugin"; + +type ResponseData = + | { + completed: boolean; + jobStatus: DeferredSplitgraphImportTask["response"]["jobStatus"]; + } + | { error: string; completed: false }; + +/** + * To manually send a request, example: + +```bash +curl -i \ + -H "Content-Type: application/json" http://localhost:3000/api/await-import-from-github \ + -d '{ "taskId": "xxxx", "splitgraphNamespace": "xxx", "splitgraphRepo": "yyy" }' +``` + */ +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + const missing = [ + "taskId", + "splitgraphNamespace", + "splitgraphRepository", + ].filter((expKey) => !req.body[expKey]); + if (missing.length > 0) { + res.status(400).json({ + error: `Missing required keys: ${missing.join(", ")}`, + completed: false, + }); + return; + } + + const { taskId, splitgraphNamespace, splitgraphRepository } = req.body; + + try { + const maybeCompletedTask = await pollImport({ + splitgraphTaskId: taskId, + splitgraphDestinationNamespace: splitgraphNamespace, + splitgraphDestinationRepository: splitgraphRepository, + }); + + if (maybeCompletedTask.error) { + throw new Error(JSON.stringify(maybeCompletedTask.error)); + } + + res.status(200).json(maybeCompletedTask); + return; + } catch (err) { + res.status(400).json({ + error: err.message, + completed: false, + }); + return; + } +} + +const pollImport = async ({ + splitgraphTaskId, + splitgraphDestinationNamespace, + splitgraphDestinationRepository, +}: { + splitgraphDestinationNamespace: string; + splitgraphDestinationRepository: string; + splitgraphTaskId: string; +}) => { + const db = makeAuthenticatedSplitgraphDb(); + + // NOTE: We must call this, or else requests will fail silently + await db.fetchAccessToken(); + + const maybeCompletedTask = (await db.pollDeferredTask("csv", { + taskId: splitgraphTaskId, + namespace: splitgraphDestinationNamespace, + repository: splitgraphDestinationRepository, + })) as DeferredSplitgraphImportTask; + + // NOTE: We do not include the jobLog, in case it could leak the GitHub PAT + // (remember we're using our PAT on behalf of the users of this app) + return { + completed: maybeCompletedTask?.completed ?? false, + jobStatus: maybeCompletedTask?.response.jobStatus, + error: maybeCompletedTask?.error ?? undefined, + }; +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts new file mode 100644 index 0000000..01c2b58 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts @@ -0,0 +1,117 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { + makeAuthenticatedSplitgraphDb, + claimsFromJWT, +} from "../../lib-backend/splitgraph-db"; + +const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET; + +type ResponseData = + | { + destination: { + splitgraphNamespace: string; + splitgraphRepository: string; + }; + taskId: string; + } + | { error: string }; + +/** + * To manually send a request, example: + +```bash +curl -i \ + -H "Content-Type: application/json" http://localhost:3000/api/start-import-from-github \ + -d '{ "githubSourceRepository": "splitgraph/seafowl", "splitgraphDestinationRepository": "import-via-nextjs" }' +``` + */ +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + const db = makeAuthenticatedSplitgraphDb(); + const { username } = claimsFromJWT((await db.fetchAccessToken()).token); + + const { githubSourceRepository } = req.body; + + if (!githubSourceRepository) { + res.status(400).json({ error: "githubSourceRepository is required" }); + return; + } + + const splitgraphDestinationRepository = + req.body.splitgraphDestinationRepository ?? + `github-import-${githubSourceRepository.replaceAll("/", "-")}`; + + try { + const taskId = await startImport({ + db, + githubSourceRepository, + splitgraphDestinationRepository, + githubStartDate: req.body.githubStartDate, + }); + res.status(200).json({ + destination: { + splitgraphNamespace: username, + splitgraphRepository: splitgraphDestinationRepository, + }, + taskId, + }); + } catch (err) { + res.status(400).json({ + error: err.message, + }); + } +} + +const startImport = async ({ + db, + githubSourceRepository, + splitgraphDestinationRepository, + githubStartDate, +}: { + db: ReturnType; + githubSourceRepository: string; + splitgraphDestinationRepository: string; + /** + * Optional start date for ingestion, must be in format like: 2021-06-01T00:00:00Z + * Defaults to 2020-01-01T00:00:00Z + * */ + githubStartDate?: string; +}) => { + const { username: splitgraphNamespace } = claimsFromJWT( + (await db.fetchAccessToken()).token + ); + + const { taskId } = await db.importData( + "airbyte-github", + { + credentials: { + credentials: { + personal_access_token: GITHUB_PAT_SECRET, + }, + }, + params: { + repository: githubSourceRepository, + start_date: githubStartDate ?? "2020-01-01T00:00:00Z", + }, + }, + { + namespace: splitgraphNamespace, + repository: splitgraphDestinationRepository, + tables: [ + { + name: "stargazers", + options: { + airbyte_cursor_field: ["starred_at"], + airbyte_primary_key_field: [], + }, + schema: [], + }, + ], + }, + { defer: true } + ); + + return taskId; +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json index 16bb209..6446b1b 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json +++ b/examples/nextjs-import-airbyte-github-export-seafowl/tsconfig.json @@ -15,6 +15,6 @@ "isolatedModules": true, "jsx": "preserve" }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], + "include": ["next-env.d.ts", "env-vars.d.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] } From 82db03bd0590564f53d9475a59840eee2b88cbb5 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Wed, 24 May 2023 20:30:11 +0100 Subject: [PATCH 04/36] Add Stepper component with ImportPanel and ExportPanel Implement the import panel and stub out the export panel, using a single Stepper component and a react context with a reducer for managing the state. Implement the fetch requests to start the import, and also to await the import. Co-Authored by GPT-4 ;) --- .../components/BaseLayout.module.css | 1 + .../ExportPanel.module.css | 6 + .../ImportExportStepper/ExportPanel.tsx | 19 ++++ .../ImportExportStepper/ImportLoadingBar.tsx | 74 ++++++++++++ .../ImportPanel.module.css | 10 ++ .../ImportExportStepper/ImportPanel.tsx | 105 ++++++++++++++++++ .../ImportExportStepper/Stepper.module.css | 4 + .../ImportExportStepper/Stepper.tsx | 16 +++ .../ImportExportStepper/StepperContext.tsx | 34 ++++++ .../ImportExportStepper/stepper-states.ts | 93 ++++++++++++++++ .../components/global-styles/theme.css | 1 + .../pages/index.tsx | 54 +-------- 12 files changed, 366 insertions(+), 51 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css index 5cee596..76f89c8 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css @@ -34,4 +34,5 @@ /* add additional styles for your content area */ color: var(--text); background-color: var(--background); + padding: 24px; } diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css new file mode 100644 index 0000000..4a5ea5b --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css @@ -0,0 +1,6 @@ +/* ExportPanel.module.css */ + +.exportPanel { + /* Style for export panel will go here */ + background: inherit; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx new file mode 100644 index 0000000..903810b --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx @@ -0,0 +1,19 @@ +import styles from "./ExportPanel.module.css"; +import { useStepper } from "./StepperContext"; + +export const ExportPanel = () => { + const [{ stepperState }] = useStepper(); + + const disabled = + stepperState !== "import_complete" && + stepperState !== "awaiting_export" && + stepperState !== "export_complete"; + + // We will fill this in later + + return ( +
+ {disabled ? "Export disabled" : "Export..."} +
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx new file mode 100644 index 0000000..8130fed --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx @@ -0,0 +1,74 @@ +import { useEffect } from "react"; +import { useStepper } from "./StepperContext"; + +type ImportLoadingBarProps = { + taskId: string; + splitgraphNamespace: string; + splitgraphRepository: string; +}; + +export const ImportLoadingBar: React.FC = ({ + taskId, + splitgraphNamespace, + splitgraphRepository, +}) => { + const [{ stepperState }, dispatch] = useStepper(); + + useEffect(() => { + if (!taskId || !splitgraphNamespace || !splitgraphRepository) { + console.log("Don't check import until we have all the right variables"); + console.table({ + taskId: taskId ?? "no task id", + splitgraphNamespace: splitgraphNamespace ?? "no namespace", + splitgraphRepository: splitgraphRepository ?? "no repo", + }); + return; + } + + if (stepperState !== "awaiting_import") { + console.log("Done waiting"); + return; + } + + const checkImportStatus = async () => { + try { + const response = await fetch("/api/await-import-from-github", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + taskId, + splitgraphNamespace, + splitgraphRepository, + }), + }); + const data = await response.json(); + + if (data.completed) { + dispatch({ type: "import_complete" }); + } else if (data.error) { + dispatch({ type: "import_error", error: data.error }); + } + } catch (error) { + console.error("Error occurred during import task status check:", error); + dispatch({ + type: "import_error", + error: "An error occurred during the import process", + }); + } + }; + + const interval = setInterval(checkImportStatus, 3000); + + return () => clearInterval(interval); + }, [ + stepperState, + taskId, + splitgraphNamespace, + splitgraphRepository, + dispatch, + ]); + + return
Loading...
; +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css new file mode 100644 index 0000000..ea3961d --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.module.css @@ -0,0 +1,10 @@ +.importPanel { + background: inherit; +} + +.error { + background-color: var(--danger); + padding: 8px; + border: 1px solid var(--sidebar); + margin-bottom: 8px; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx new file mode 100644 index 0000000..7da3725 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx @@ -0,0 +1,105 @@ +import { useState } from "react"; +import { useStepper } from "./StepperContext"; +import { ImportLoadingBar } from "./ImportLoadingBar"; + +import styles from "./ImportPanel.module.css"; + +export const ImportPanel = () => { + const [ + { stepperState, taskId, error, splitgraphNamespace, splitgraphRepository }, + dispatch, + ] = useStepper(); + const [inputValue, setInputValue] = useState(""); + + const handleInputSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + + if (!isValidRepoName(inputValue)) { + dispatch({ + type: "import_error", + error: + "Invalid GitHub repository name. Format must be 'namespace/repository'", + }); + return; + } + + const [githubNamespace, githubRepository] = inputValue.split("/"); + + try { + const response = await fetch(`/api/start-import-from-github`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ githubSourceRepository: inputValue }), + }); + + if (!response.ok) { + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + + if (!data.taskId) { + throw new Error("Response missing taskId"); + } + + if (!data.destination || !data.destination.splitgraphNamespace) { + throw new Error("Response missing destination.splitgraphNamespace"); + } + + if (!data.destination || !data.destination.splitgraphRepository) { + throw new Error("Response missing destination.splitgraphRepository"); + } + + dispatch({ + type: "start_import", + repository: { + namespace: githubNamespace, + repository: githubRepository, + }, + taskId: data.taskId, + splitgraphRepository: data.destination.splitgraphRepository as string, + splitgraphNamespace: data.destination.splitgraphNamespace as string, + }); + } catch (error) { + dispatch({ type: "import_error", error: error.message }); + } + }; + + const isValidRepoName = (repoName: string) => { + // A valid GitHub repo name should contain exactly one '/' + return /^[\w-.]+\/[\w-.]+$/.test(repoName); + }; + + return ( +
+ {stepperState === "unstarted" && ( + <> + {error &&

{error}

} +
+ setInputValue(e.target.value)} + /> + +
+ + )} + {stepperState === "awaiting_import" && ( + + )} + {stepperState === "import_complete" && ( +
+

Import Complete

+
+ )} +
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css new file mode 100644 index 0000000..a1eb9c2 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.module.css @@ -0,0 +1,4 @@ +.stepper { + /* Add styling as necessary */ + background: inherit; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx new file mode 100644 index 0000000..6af0f06 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx @@ -0,0 +1,16 @@ +import { StepperContextProvider } from "./StepperContext"; +import { ImportPanel } from "./ImportPanel"; // will create this component later +import { ExportPanel } from "./ExportPanel"; // will create this component later + +import styles from "./Stepper.module.css"; + +export const Stepper = () => { + return ( + +
+ + +
+ + ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx new file mode 100644 index 0000000..cbff3a1 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx @@ -0,0 +1,34 @@ +// StepperContext.tsx +import React, { useReducer, useContext, ReactNode } from "react"; +import { + StepperState, + StepperAction, + initialState, + stepperReducer, +} from "./stepper-states"; + +// Define the context +const StepperContext = React.createContext< + [StepperState, React.Dispatch] | undefined +>(undefined); + +export const StepperContextProvider: React.FC<{ children: ReactNode }> = ({ + children, +}) => { + const [state, dispatch] = useReducer(stepperReducer, initialState); + + return ( + + {children} + + ); +}; + +// Custom hook for using the stepper context +export const useStepper = () => { + const context = useContext(StepperContext); + if (!context) { + throw new Error("useStepper must be used within a StepperContextProvider"); + } + return context; +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts new file mode 100644 index 0000000..180c0fb --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts @@ -0,0 +1,93 @@ +// stepper-states.ts +export type GitHubRepository = { namespace: string; repository: string }; + +// Define the state +export type StepperState = { + stepperState: + | "unstarted" + | "awaiting_import" + | "import_complete" + | "awaiting_export" + | "export_complete"; + repository?: GitHubRepository | null; + taskId?: string | null; + error?: string; + tables?: { taskId: string }[] | null; + splitgraphRepository?: string; + splitgraphNamespace?: string; +}; + +// Define the actions +export type StepperAction = + | { + type: "start_import"; + repository: GitHubRepository; + taskId: string; + splitgraphRepository: string; + splitgraphNamespace: string; + } + | { type: "import_complete" } + | { type: "start_export"; tables: { taskId: string }[] } + | { type: "export_complete" } + | { type: "import_error"; error: string } + | { type: "reset" }; + +// Initial state +export const initialState: StepperState = { + stepperState: "unstarted", + repository: null, + splitgraphRepository: null, + splitgraphNamespace: null, + taskId: null, + tables: null, +}; + +// Reducer function +export const stepperReducer = ( + state: StepperState, + action: StepperAction +): StepperState => { + console.log("Got action", action, "prev state:", state); + switch (action.type) { + case "start_import": + return { + ...state, + stepperState: "awaiting_import", + repository: action.repository, + taskId: action.taskId, + splitgraphNamespace: action.splitgraphNamespace, + splitgraphRepository: action.splitgraphRepository, + }; + case "import_complete": + return { + ...state, + stepperState: "import_complete", + }; + case "start_export": + return { + ...state, + stepperState: "awaiting_export", + tables: action.tables, + }; + case "export_complete": + return { + ...state, + stepperState: "export_complete", + }; + case "import_error": + return { + ...state, + splitgraphRepository: null, + splitgraphNamespace: null, + taskId: null, + stepperState: "unstarted", + error: action.error, + }; + + case "reset": + return initialState; + + default: + return state; + } +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css index ecf2ea9..66c4514 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/global-styles/theme.css @@ -7,6 +7,7 @@ --sidebar: #718096; --text: #1a202c; --subtext: #718096; + --danger: #eb8585; } body { diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx index bb79aff..4218207 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/index.tsx @@ -1,58 +1,10 @@ -import { - SqlProvider, - useSql, - makeSplitgraphHTTPContext, -} from "@madatdata/react"; +import { SqlProvider, makeSplitgraphHTTPContext } from "@madatdata/react"; import { useMemo } from "react"; import { BaseLayout } from "../components/BaseLayout"; import { Sidebar, type GitHubRepository } from "../components/Sidebar"; - -const ExampleComponentUsingSQL = () => { - const { loading, error, response } = useSql<{ - origin_airport: string; - destination_airport: string; - origin_city: string; - destination_city: string; - passengers: number; - seats: number; - flights: number; - distance: number; - fly_month: string; - origin_pop: number; - destination_pop: number; - id: number; - }>( - `SELECT - "origin_airport", - "destination_airport", - "origin_city", - "destination_city", - "passengers", - "seats", - "flights", - "distance", - "fly_month", - "origin_pop", - "destination_pop", - "id" -FROM - "splitgraph/domestic_us_flights:latest"."flights" -LIMIT 100;` - ); - - return ( - 
-      {JSON.stringify({ loading, error, response }, null, 2)}
-
- ); -}; +import { Stepper } from "../components/ImportExportStepper/Stepper"; const SplitgraphSampleQuery = () => { const splitgraphDataContext = useMemo( @@ -64,7 +16,7 @@ const SplitgraphSampleQuery = () => { return ( }> - + ); From 09750751bb3deac74c19bc68ffe1b5869b2d2975 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Thu, 25 May 2023 04:00:22 +0100 Subject: [PATCH 05/36] Implement backend API routes `start-export-to-seafowl` and `await-export-to-seafowl-task` The `start-export-to-seafowl` route takes a list of source tables from Splitgraph (list of `{namespace,repository,table}`), and starts a task to export them to Seafowl. It returns a list of objects `{taskId: string; tableName: string;}`, where each item represents the currently exporting table (and `tableName` is the source table name). The `await-export-to-seafowl-task` route takes a single `taskId` parameter and returns its status, i.e. `{completed: boolean; ...otherInfo}` --- .../pages/api/await-export-to-seafowl-task.ts | 76 +++++++++++++ .../pages/api/start-export-to-seafowl.ts | 103 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts new file mode 100644 index 0000000..93f6b89 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts @@ -0,0 +1,76 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db"; +import type { DeferredSplitgraphExportTask } from "@madatdata/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin"; + +type ResponseData = + | { + completed: boolean; + jobStatus: DeferredSplitgraphExportTask["response"]; + } + | { error: string; completed: false }; + +/** + * To manually send a request, example: + +```bash +curl -i \ + -H "Content-Type: application/json" http://localhost:3000/api/await-export-to-seafowl-task \ + -d '{ "taskId": "2923fd6f-2197-495a-9df1-2428a9ca8dee" }' +``` + */ +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + if (!req.body["taskId"]) { + res.status(400).json({ + error: "Missing required key: taskId", + completed: false, + }); + return; + } + + const { taskId } = req.body; + + try { + const maybeCompletedTask = await pollImport({ + splitgraphTaskId: taskId, + }); + + if (maybeCompletedTask.error) { + throw new Error(JSON.stringify(maybeCompletedTask.error)); + } + + res.status(200).json(maybeCompletedTask); + return; + } catch (err) { + res.status(400).json({ + error: err.message, + completed: false, + }); + return; + } +} + +const pollImport = async ({ + splitgraphTaskId, +}: { + splitgraphTaskId: string; +}) => { + const db = makeAuthenticatedSplitgraphDb(); + + // NOTE: We must call this, or else requests will fail silently + await db.fetchAccessToken(); + + const maybeCompletedTask = (await db.pollDeferredTask("export-to-seafowl", { + taskId: splitgraphTaskId, + })) as DeferredSplitgraphExportTask; + + // NOTE: We do not include the jobLog, in case it could leak the GitHub PAT + // (remember we're using our PAT on behalf of the users of this app) + return { + completed: maybeCompletedTask?.completed ?? false, + jobStatus: maybeCompletedTask?.response, + error: maybeCompletedTask?.error ?? undefined, + }; +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts new file mode 100644 index 0000000..75d6343 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts @@ -0,0 +1,103 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db"; + +type ResponseData = + | { + tables: { + tableName: string; + taskId: string; + }[]; + } + | { error: string }; + +type TableInput = { namespace: string; repository: string; table: string }; + +/** + * To manually send a request, example: + +```bash +curl -i \ + -H "Content-Type: application/json" http://localhost:3000/api/start-export-to-seafowl \ + -d '{ "tables": [{"namespace": "miles", "repository": "import-via-nextjs", "table": "stargazers"}] }' +``` + */ +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + const db = makeAuthenticatedSplitgraphDb(); + const { tables } = req.body; + + if ( + !tables || + !tables.length || + !tables.every( + (t: TableInput) => + t.namespace && + t.repository && + t.table && + typeof t.namespace === "string" && + typeof t.repository === "string" && + typeof t.table === "string" + ) + ) { + res.status(400).json({ error: "invalid tables input in request body" }); + return; + } + + try { + const exportingTables = await startExport({ + db, + tables, + }); + res.status(200).json({ + tables: exportingTables, + }); + } catch (err) { + res.status(400).json({ + error: err.message, + }); + } +} + +const startExport = async ({ + db, + tables, +}: { + db: ReturnType; + tables: TableInput[]; +}) => { + await db.fetchAccessToken(); + + const response = await db.exportData( + "export-to-seafowl", + { + tables: tables.map((splitgraphSource) => ({ + source: { + repository: splitgraphSource.repository, + namespace: splitgraphSource.namespace, + table: splitgraphSource.table, + }, + })), + }, + { + // Empty instance will trigger Splitgraph to export to demo.seafowl.cloud + seafowlInstance: {}, + }, + { defer: true } + ); + + if (response.error) { + throw new Error(JSON.stringify(response.error)); + } + + const loadingTables: { taskId: string; tableName: string }[] = + response.taskIds.tables.map( + (t: { jobId: string; sourceTable: string }) => ({ + taskId: t.jobId, + tableName: t.sourceTable, + }) + ); + + return loadingTables; +}; From 5cd10be02a86d6da9d27dbf79a5fb8689e84e502 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Thu, 25 May 2023 04:02:19 +0100 Subject: [PATCH 06/36] Implement the components for the "Export Panel" The ExportPanel first renders a "Start Export" button. Then, while the export is running, it renders an `ExportTableLoadingBar` for each table that is being exported. Each of thee individual components sends its own polling request with its `taskId` to the `await-export-to-seafowl-task` endpoint, and upon completion of each task, sends an action to the reducer, which handles it by updating the set of loading tasks. When the set of loading tasks is complete, it changes the `stepperState` to `export_complete`. If any of the tasks has an error, then the `stepperState` changes to `export_error` which should cause all loading bars to unmount - i.e., any error will short-circuit all of the table loading, even if some were to complete. At that point the user can click "start export" again. This completes the logic necessary for import and export, and now it's just a matter of styling the components, linking to the Splitgraph Console, adding explanatory text, and finally rendering a chart with the data. We'll also want to create a meta-repository in Splitgraph for tracking which GitHub repos we've imported so far, analogously to how we track Socrata metadata for each Socrata repo. --- .../ExportLoadingBars.module.css | 3 + .../ImportExportStepper/ExportLoadingBars.tsx | 19 +++++ .../ExportPanel.module.css | 19 ++++- .../ImportExportStepper/ExportPanel.tsx | 74 +++++++++++++++-- .../ExportTableLoadingBar.module.css | 17 ++++ .../ExportTableLoadingBar.tsx | 78 +++++++++++++++++ .../ImportExportStepper/ImportLoadingBar.tsx | 1 - .../ImportExportStepper/ImportPanel.tsx | 12 ++- .../ImportExportStepper/stepper-states.ts | 83 +++++++++++++++---- 9 files changed, 277 insertions(+), 29 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css new file mode 100644 index 0000000..04c119f --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.module.css @@ -0,0 +1,3 @@ +.exportLoadingBars { + background-color: inherit; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx new file mode 100644 index 0000000..f830456 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx @@ -0,0 +1,19 @@ +import { useStepper } from "./StepperContext"; +import { ExportTableLoadingBar } from "./ExportTableLoadingBar"; +import styles from "./ExportLoadingBars.module.css"; + +export const ExportLoadingBars = () => { + const [{ exportedTablesLoading }] = useStepper(); + + return ( +
+ {Array.from(exportedTablesLoading).map(({ tableName, taskId }) => ( + + ))} +
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css index 4a5ea5b..c82f97c 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.module.css @@ -1,6 +1,19 @@ -/* ExportPanel.module.css */ - .exportPanel { - /* Style for export panel will go here */ + /* Styles for the export panel container */ + background: inherit; +} + +.startExportButton { + /* Styles for the start export button */ + background: inherit; +} + +.querySeafowlButton { + /* Styles for the query Seafowl button */ + background: inherit; +} + +.viewReportButton { + /* Styles for the view report button */ background: inherit; } diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx index 903810b..8152602 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx @@ -1,19 +1,77 @@ -import styles from "./ExportPanel.module.css"; +// components/ImportExportStepper/ExportPanel.tsx + import { useStepper } from "./StepperContext"; +import styles from "./ExportPanel.module.css"; +import { ExportLoadingBars } from "./ExportLoadingBars"; + +// TODO: don't hardcode this? or at least hardcode all of them and make it official +const importedTableNames = [ + "stargazers", + // NOTE: If we only specify stargazers, then stargazers_user is still included since it's a dependent table + "stargazers_user", +]; export const ExportPanel = () => { - const [{ stepperState }] = useStepper(); + const [ + { stepperState, exportError, splitgraphRepository, splitgraphNamespace }, + dispatch, + ] = useStepper(); + + const handleStartExport = async () => { + try { + const response = await fetch("/api/start-export-to-seafowl", { + method: "POST", + body: JSON.stringify({ + tables: importedTableNames.map((tableName) => ({ + namespace: splitgraphNamespace, + repository: splitgraphRepository, + table: tableName, + })), + }), + headers: { + "Content-Type": "application/json", + }, + }); + const data = await response.json(); - const disabled = - stepperState !== "import_complete" && - stepperState !== "awaiting_export" && - stepperState !== "export_complete"; + if (!data.tables || !data.tables.length) { + throw new Error("Response missing tables"); + } - // We will fill this in later + dispatch({ + type: "start_export", + tables: data.tables.map( + ({ tableName, taskId }: { tableName: string; taskId: string }) => ({ + taskId, + tableName, + }) + ), + }); + } catch (error) { + dispatch({ type: "export_error", error: error.message }); + } + }; return (
- {disabled ? "Export disabled" : "Export..."} + {exportError &&

{exportError}

} + {stepperState === "import_complete" && ( + + )} + {stepperState === "awaiting_export" && } + {stepperState === "export_complete" && ( + <> + + + + )}
); }; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css new file mode 100644 index 0000000..9f75de7 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.module.css @@ -0,0 +1,17 @@ +/* components/ImportExportStepper/ExportTableLoadingBar.module.css */ + +.exportTableLoadingBar { + background-color: inherit; +} + +.loadingBar { + background-color: inherit; +} + +.completedBar { + background-color: inherit; +} + +.tableName { + background-color: inherit; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx new file mode 100644 index 0000000..76e8799 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx @@ -0,0 +1,78 @@ +import { useEffect } from "react"; +import { useStepper } from "./StepperContext"; +import styles from "./ExportTableLoadingBar.module.css"; + +interface ExportTableLoadingBarProps { + tableName: string; + taskId: string; +} + +export const ExportTableLoadingBar = ({ + tableName, + taskId, +}: React.PropsWithoutRef) => { + const [{ stepperState, exportedTablesLoading }, dispatch] = useStepper(); + + useEffect(() => { + if (!taskId || !tableName) { + console.log("Don't check export until we have taskId and tableName"); + console.table({ + taskId, + tableName, + }); + return; + } + + if (stepperState !== "awaiting_export") { + console.log("Done waiting for export"); + return; + } + + const pollExportTask = async () => { + try { + const response = await fetch("/api/await-export-to-seafowl-task", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + taskId, + }), + }); + const data = await response.json(); + + if (data.completed) { + dispatch({ + type: "export_table_task_complete", + completedTable: { tableName, taskId }, + }); + } else if (data.error) { + throw new Error(data.error); + } + } catch (error) { + dispatch({ + type: "export_error", + error: `Error exporting ${tableName}: ${error.message}`, + }); + } + }; + + const interval = setInterval(pollExportTask, 3000); + return () => clearInterval(interval); + }, [stepperState, tableName, taskId, dispatch]); + + const isLoading = !!Array.from(exportedTablesLoading).find( + (t) => t.taskId === taskId + ); + + return ( +
+
+ {isLoading + ? `Loading ${tableName}...` + : `Successfully exported ${tableName}`} +
+
{tableName}
+
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx index 8130fed..3795a61 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportLoadingBar.tsx @@ -51,7 +51,6 @@ export const ImportLoadingBar: React.FC = ({ dispatch({ type: "import_error", error: data.error }); } } catch (error) { - console.error("Error occurred during import task status check:", error); dispatch({ type: "import_error", error: "An error occurred during the import process", diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx index 7da3725..b57f2ef 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ImportPanel.tsx @@ -6,7 +6,13 @@ import styles from "./ImportPanel.module.css"; export const ImportPanel = () => { const [ - { stepperState, taskId, error, splitgraphNamespace, splitgraphRepository }, + { + stepperState, + importTaskId, + importError, + splitgraphNamespace, + splitgraphRepository, + }, dispatch, ] = useStepper(); const [inputValue, setInputValue] = useState(""); @@ -76,7 +82,7 @@ export const ImportPanel = () => {
{stepperState === "unstarted" && ( <> - {error &&

{error}

} + {importError &&

{importError}

}
{ )} {stepperState === "awaiting_import" && ( diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts index 180c0fb..c7621a3 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts @@ -1,7 +1,8 @@ // stepper-states.ts export type GitHubRepository = { namespace: string; repository: string }; -// Define the state +type ExportTable = { tableName: string; taskId: string }; + export type StepperState = { stepperState: | "unstarted" @@ -10,14 +11,15 @@ export type StepperState = { | "awaiting_export" | "export_complete"; repository?: GitHubRepository | null; - taskId?: string | null; - error?: string; - tables?: { taskId: string }[] | null; + importTaskId?: string | null; + importError?: string; splitgraphRepository?: string; splitgraphNamespace?: string; + exportedTablesLoading?: Set; + exportedTablesCompleted?: Set; + exportError?: string; }; -// Define the actions export type StepperAction = | { type: "start_import"; @@ -27,34 +29,45 @@ export type StepperAction = splitgraphNamespace: string; } | { type: "import_complete" } - | { type: "start_export"; tables: { taskId: string }[] } + | { type: "start_export"; tables: ExportTable[] } + | { type: "export_table_task_complete"; completedTable: ExportTable } | { type: "export_complete" } + | { type: "export_error"; error: string } | { type: "import_error"; error: string } | { type: "reset" }; -// Initial state export const initialState: StepperState = { stepperState: "unstarted", repository: null, splitgraphRepository: null, splitgraphNamespace: null, - taskId: null, - tables: null, + importTaskId: null, + exportedTablesLoading: new Set(), + exportedTablesCompleted: new Set(), + importError: null, + exportError: null, }; +// FOR DEBUGGING: uncomment for hardcoded state initialization +// export const initialState: StepperState = { +// ...normalInitialState, +// stepperState: "import_complete", +// splitgraphNamespace: "miles", +// splitgraphRepository: "import-via-nextjs", +// }; + // Reducer function export const stepperReducer = ( state: StepperState, action: StepperAction ): StepperState => { - console.log("Got action", action, "prev state:", state); switch (action.type) { case "start_import": return { ...state, stepperState: "awaiting_import", repository: action.repository, - taskId: action.taskId, + importTaskId: action.taskId, splitgraphNamespace: action.splitgraphNamespace, splitgraphRepository: action.splitgraphRepository, }; @@ -64,11 +77,45 @@ export const stepperReducer = ( stepperState: "import_complete", }; case "start_export": + const { tables } = action; + const exportedTablesLoading = new Set(); + const exportedTablesCompleted = new Set(); + + for (const { tableName, taskId } of tables) { + exportedTablesLoading.add({ tableName, taskId }); + } + return { ...state, + exportedTablesLoading, + exportedTablesCompleted, stepperState: "awaiting_export", - tables: action.tables, }; + + case "export_table_task_complete": + const { completedTable } = action; + + // We're storing a set of completedTable objects, so we need to find the matching one to remove it + const loadingTablesAfterRemoval = new Set(state.exportedTablesLoading); + const loadingTabletoRemove = Array.from(loadingTablesAfterRemoval).find( + ({ taskId }) => taskId === completedTable.taskId + ); + loadingTablesAfterRemoval.delete(loadingTabletoRemove); + + // Then we can add the matching one to the completed table + const completedTablesAfterAdded = new Set(state.exportedTablesCompleted); + completedTablesAfterAdded.add(completedTable); + + return { + ...state, + exportedTablesLoading: loadingTablesAfterRemoval, + exportedTablesCompleted: completedTablesAfterAdded, + stepperState: + loadingTablesAfterRemoval.size === 0 + ? "export_complete" + : "awaiting_export", + }; + case "export_complete": return { ...state, @@ -79,9 +126,17 @@ export const stepperReducer = ( ...state, splitgraphRepository: null, splitgraphNamespace: null, - taskId: null, + importTaskId: null, stepperState: "unstarted", - error: action.error, + importError: action.error, + }; + case "export_error": + return { + ...state, + exportedTablesLoading: new Set(), + exportedTablesCompleted: new Set(), + stepperState: "import_complete", + exportError: action.error, }; case "reset": From 8ba415160b343ea45703b4b8106c6194054a8aa8 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Fri, 26 May 2023 23:36:19 +0100 Subject: [PATCH 07/36] Move lib-backend to lib/backend --- .../{lib-backend => lib/backend}/splitgraph-db.ts | 0 .../pages/api/await-export-to-seafowl-task.ts | 2 +- .../pages/api/await-import-from-github.ts | 2 +- .../pages/api/start-export-to-seafowl.ts | 2 +- .../pages/api/start-import-from-github.ts | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename examples/nextjs-import-airbyte-github-export-seafowl/{lib-backend => lib/backend}/splitgraph-db.ts (100%) diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts similarity index 100% rename from examples/nextjs-import-airbyte-github-export-seafowl/lib-backend/splitgraph-db.ts rename to examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts index 93f6b89..61a7ae1 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-export-to-seafowl-task.ts @@ -1,5 +1,5 @@ import type { NextApiRequest, NextApiResponse } from "next"; -import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db"; +import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db"; import type { DeferredSplitgraphExportTask } from "@madatdata/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin"; type ResponseData = diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts index d06aac9..4563044 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/await-import-from-github.ts @@ -1,5 +1,5 @@ import type { NextApiRequest, NextApiResponse } from "next"; -import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db"; +import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db"; import type { DeferredSplitgraphImportTask } from "@madatdata/db-splitgraph/plugins/importers/splitgraph-base-import-plugin"; type ResponseData = diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts index 75d6343..4e95752 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts @@ -1,5 +1,5 @@ import type { NextApiRequest, NextApiResponse } from "next"; -import { makeAuthenticatedSplitgraphDb } from "../../lib-backend/splitgraph-db"; +import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db"; type ResponseData = | { diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts index 01c2b58..147aeba 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { makeAuthenticatedSplitgraphDb, claimsFromJWT, -} from "../../lib-backend/splitgraph-db"; +} from "../../lib/backend/splitgraph-db"; const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET; From 2dcec66809ccc1b096f44dee601bda5118d119f8 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Fri, 26 May 2023 23:58:26 +0100 Subject: [PATCH 08/36] Hardcode list of "relevant" table names for ingestion from GitHub The `airbyte-github` plugin by default imports 163 tables into Splitgraph, but we only need a few of them for the analytics queries we want to make in the demo app. So, hardcode the list of those, but also hardcode the list of all 163 tables for reference, and also the 43 tables that are imported given the relevant tables (because either they depend on them via a foreign key relationship, or they're an airbyte meta table). For the 43 tables, see this recent import of `splitgraph/seafowl`: * https://www.splitgraph.com/miles/github-import-splitgraph-seafowl/20230526-224723/-/tables This took 3 minutes and 40 seconds to import into Splitgraph. --- .../ImportExportStepper/ExportPanel.tsx | 9 +- .../lib/config.ts | 249 ++++++++++++++++++ .../pages/api/start-import-from-github.ts | 12 +- 3 files changed, 256 insertions(+), 14 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx index 8152602..ceb29f6 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx @@ -4,12 +4,7 @@ import { useStepper } from "./StepperContext"; import styles from "./ExportPanel.module.css"; import { ExportLoadingBars } from "./ExportLoadingBars"; -// TODO: don't hardcode this? or at least hardcode all of them and make it official -const importedTableNames = [ - "stargazers", - // NOTE: If we only specify stargazers, then stargazers_user is still included since it's a dependent table - "stargazers_user", -]; +import { relevantGitHubTableNames } from "../../lib/config"; export const ExportPanel = () => { const [ @@ -22,7 +17,7 @@ export const ExportPanel = () => { const response = await fetch("/api/start-export-to-seafowl", { method: "POST", body: JSON.stringify({ - tables: importedTableNames.map((tableName) => ({ + tables: relevantGitHubTableNames.map((tableName) => ({ namespace: splitgraphNamespace, repository: splitgraphRepository, table: tableName, diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts new file mode 100644 index 0000000..8f6719a --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts @@ -0,0 +1,249 @@ +/** + * List of GitHub table names that we want to import with the Airbyte connector + * into Splitgraph. By default, there are 163 tables available. But we only want + * some of them, and by selecting them explicitly, the import will be much faster, + * especially for large repositories. + * + * Note that Airbyte will still import tables that depend on these tables due + * to foreign keys, and will also import airbyte metaata tables. + */ +export const relevantGitHubTableNames = `commits +comments +pull_requests +pull_request_stats +issue_reactions` + .split("\n") + .filter((t) => !!t); + +/** + * List of "downstream" GitHub table names that will be imported by default by + * the `airbyte-github` connector, given the list of `relevantGitHubTableNames`, + * because they're either an Airbyte meta table or a table that depends on + * one of the "relevant" tables. + * + * This is manually curated and might not be totally accurate. It's up to date + * given the following list of `relevantGitHubTableNames`: + * + * ``` + * commits + * comments + * pull_requests + * pull_request_stats + * issue_reactions + * ``` + */ +export const expectedImportedTableNames = `_airbyte_raw_comments +_airbyte_raw_commits +_airbyte_raw_issue_reactions +_airbyte_raw_pull_request_stats +_airbyte_raw_pull_requests +_sg_ingestion_state +comments +comments_user +commits +commits_author +commits_commit +commits_commit_author +commits_commit_committer +commits_commit_tree +commits_commit_verification +commits_committer +commits_parents +issue_reactions +issue_reactions_user +pull_request_stats +pull_request_stats_merged_by +pull_requests +pull_requests__links +pull_requests__links_comments +pull_requests__links_commits +pull_requests__links_html +pull_requests__links_issue +pull_requests__links_review_comment +pull_requests__links_review_comments +pull_requests__links_self +pull_requests__links_statuses +pull_requests_assignee +pull_requests_assignees +pull_requests_auto_merge +pull_requests_auto_merge_enabled_by +pull_requests_base +pull_requests_head +pull_requests_labels +pull_requests_milestone +pull_requests_milestone_creator +pull_requests_requested_reviewers +pull_requests_requested_teams +pull_requests_user +`; + +/** + * This is the list of all tables imported by Airbyte by default when no tables + * are explicitly provided to the plugin. + * + * This is not consumed anywhere, but is useful for referencing, and if you'd + * like to extend or modify the code, you can choose tables from here to include. + */ +export const allGitHubTableNames = `_airbyte_raw_assignees +_airbyte_raw_branches +_airbyte_raw_collaborators +_airbyte_raw_comments +_airbyte_raw_commit_comment_reactions +_airbyte_raw_commit_comments +_airbyte_raw_commits +_airbyte_raw_deployments +_airbyte_raw_events +_airbyte_raw_issue_comment_reactions +_airbyte_raw_issue_events +_airbyte_raw_issue_labels +_airbyte_raw_issue_milestones +_airbyte_raw_issue_reactions +_airbyte_raw_issues +_airbyte_raw_organizations +_airbyte_raw_project_cards +_airbyte_raw_project_columns +_airbyte_raw_projects +_airbyte_raw_pull_request_comment_reactions +_airbyte_raw_pull_request_commits +_airbyte_raw_pull_request_stats +_airbyte_raw_pull_requests +_airbyte_raw_releases +_airbyte_raw_repositories +_airbyte_raw_review_comments +_airbyte_raw_reviews +_airbyte_raw_stargazers +_airbyte_raw_tags +_airbyte_raw_team_members +_airbyte_raw_team_memberships +_airbyte_raw_teams +_airbyte_raw_users +_airbyte_raw_workflow_jobs +_airbyte_raw_workflow_runs +_airbyte_raw_workflows +_sg_ingestion_state +assignees +branches +branches_commit +branches_protection +branches_protection_required_status_checks +collaborators +collaborators_permissions +comments +comments_user +commit_comment_reactions +commit_comment_reactions_user +commit_comments +commit_comments_user +commits +commits_author +commits_commit +commits_commit_author +commits_commit_committer +commits_commit_tree +commits_commit_verification +commits_committer +commits_parents +deployments +deployments_creator +events +events_actor +events_org +events_repo +issue_comment_reactions +issue_comment_reactions_user +issue_events +issue_events_actor +issue_events_issue +issue_events_issue_user +issue_labels +issue_milestones +issue_milestones_creator +issue_reactions +issue_reactions_user +issues +issues_assignee +issues_assignees +issues_labels +issues_milestone +issues_milestone_creator +issues_pull_request +issues_user +organizations +organizations_plan +project_cards +project_cards_creator +project_columns +projects +projects_creator +pull_request_comment_reactions +pull_request_comment_reactions_user +pull_request_commits +pull_request_commits_author +pull_request_commits_commit +pull_request_commits_commit_author +pull_request_commits_commit_committer +pull_request_commits_commit_tree +pull_request_commits_commit_verification +pull_request_commits_committer +pull_request_commits_parents +pull_request_stats +pull_request_stats_merged_by +pull_requests +pull_requests__links +pull_requests__links_comments +pull_requests__links_commits +pull_requests__links_html +pull_requests__links_issue +pull_requests__links_review_comment +pull_requests__links_review_comments +pull_requests__links_self +pull_requests__links_statuses +pull_requests_assignee +pull_requests_assignees +pull_requests_auto_merge +pull_requests_auto_merge_enabled_by +pull_requests_base +pull_requests_head +pull_requests_labels +pull_requests_milestone +pull_requests_milestone_creator +pull_requests_requested_reviewers +pull_requests_requested_teams +pull_requests_user +releases +releases_assets +releases_author +repositories +repositories_license +repositories_owner +repositories_permissions +review_comments +review_comments__links +review_comments__links_html +review_comments__links_pull_request +review_comments__links_self +review_comments_user +reviews +reviews__links +reviews__links_html +reviews__links_pull_request +reviews_user +stargazers +stargazers_user +tags +tags_commit +team_members +team_memberships +teams +users +workflow_jobs +workflow_jobs_steps +workflow_runs +workflow_runs_head_commit +workflow_runs_head_commit_author +workflow_runs_head_commit_committer +workflow_runs_head_repository +workflow_runs_head_repository_owner +workflow_runs_repository +workflow_runs_repository_owner +workflows`; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts index 147aeba..dc78648 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts @@ -3,6 +3,7 @@ import { makeAuthenticatedSplitgraphDb, claimsFromJWT, } from "../../lib/backend/splitgraph-db"; +import { relevantGitHubTableNames } from "../../lib/config"; const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET; @@ -100,14 +101,11 @@ const startImport = async ({ namespace: splitgraphNamespace, repository: splitgraphDestinationRepository, tables: [ - { - name: "stargazers", - options: { - airbyte_cursor_field: ["starred_at"], - airbyte_primary_key_field: [], - }, + ...relevantGitHubTableNames.map((t) => ({ + name: t, + options: {}, schema: [], - }, + })), ], }, { defer: true } From 65ca23e59190d90e80781382d4d50d3b41b328fd Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Sat, 27 May 2023 00:00:39 +0100 Subject: [PATCH 09/36] Serialize the stepper state into the URL, so that awaiting can be resumed across page loads Keep track of the current stepper state (e.g. taskId, import completion, etc.) in the URL. Update the URL when the state changes, and initialize the state from the URL on page load. Note that we need to default to an "uninitialized" state, and then update the state from the URL via an `initialize_from_url` action, because the `useRouter` hook is ansynchronous, and we don't look at query parameters on the server side with `getInitialProps` or similar. Thus we can show a loading bar before showing the import form (or whatever we're showing based on the current state). This makes development easier, since after a long import we can refresh the page with the URL containing the task ID and start from there, rather than re-importing every time. And it also makes it easier for users who can refresh the page without losing progress if an import has already started (it will just poll the taskId from the URL). --- .../ImportExportStepper/DebugPanel.tsx | 13 ++ .../ExportTableLoadingBar.tsx | 6 +- .../ImportExportStepper/Stepper.tsx | 22 +- .../ImportExportStepper/StepperContext.tsx | 5 +- .../ImportExportStepper/stepper-states.ts | 198 +++++++++++++++++- 5 files changed, 230 insertions(+), 14 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx new file mode 100644 index 0000000..d7306b4 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/DebugPanel.tsx @@ -0,0 +1,13 @@ +import { useStepper } from "./StepperContext"; + +export const DebugPanel = () => { + const [state, _] = useStepper(); + + return ( +
+ 
+        {JSON.stringify(state, null, 2)}
+
+
+ ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx index 76e8799..7551b58 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx @@ -47,7 +47,11 @@ export const ExportTableLoadingBar = ({ completedTable: { tableName, taskId }, }); } else if (data.error) { - throw new Error(data.error); + if (!data.completed) { + console.log("WARN: Failed status, not completed:", data.error); + } else { + throw new Error(data.error); + } } } catch (error) { dispatch({ diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx index 6af0f06..93f987f 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx @@ -1,15 +1,27 @@ -import { StepperContextProvider } from "./StepperContext"; -import { ImportPanel } from "./ImportPanel"; // will create this component later -import { ExportPanel } from "./ExportPanel"; // will create this component later +import { StepperContextProvider, useStepper } from "./StepperContext"; +import { DebugPanel } from "./DebugPanel"; +import { ImportPanel } from "./ImportPanel"; +import { ExportPanel } from "./ExportPanel"; import styles from "./Stepper.module.css"; +const StepperOrLoading = ({ children }: { children: React.ReactNode }) => { + const [{ stepperState }] = useStepper(); + + return ( + <>{stepperState === "uninitialized" ?
........
: children} + ); +}; + export const Stepper = () => { return (
- - + + + + +
); diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx index cbff3a1..0fdb7c0 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx @@ -3,8 +3,7 @@ import React, { useReducer, useContext, ReactNode } from "react"; import { StepperState, StepperAction, - initialState, - stepperReducer, + useStepperReducer, } from "./stepper-states"; // Define the context @@ -15,7 +14,7 @@ const StepperContext = React.createContext< export const StepperContextProvider: React.FC<{ children: ReactNode }> = ({ children, }) => { - const [state, dispatch] = useReducer(stepperReducer, initialState); + const [state, dispatch] = useStepperReducer(); return ( diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts index c7621a3..c8fdaf1 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts @@ -1,10 +1,13 @@ -// stepper-states.ts +import { useRouter, type NextRouter } from "next/router"; +import { ParsedUrlQuery } from "querystring"; +import { useEffect, useReducer } from "react"; export type GitHubRepository = { namespace: string; repository: string }; type ExportTable = { tableName: string; taskId: string }; export type StepperState = { stepperState: + | "uninitialized" | "unstarted" | "awaiting_import" | "import_complete" @@ -34,9 +37,15 @@ export type StepperAction = | { type: "export_complete" } | { type: "export_error"; error: string } | { type: "import_error"; error: string } - | { type: "reset" }; + | { type: "reset" } + | { type: "initialize_from_url"; parsedFromUrl: StepperState }; -export const initialState: StepperState = { +type ExtractStepperAction = Extract< + StepperAction, + { type: T } +>; + +const initialState: StepperState = { stepperState: "unstarted", repository: null, splitgraphRepository: null, @@ -56,8 +65,128 @@ export const initialState: StepperState = { // splitgraphRepository: "import-via-nextjs", // }; -// Reducer function -export const stepperReducer = ( +type ActionParams = Omit< + ExtractStepperAction, + "type" +>; + +const getQueryParamAsString = ( + query: ParsedUrlQuery, + key: string +): T | null => { + if (Array.isArray(query[key]) && query[key].length > 0) { + throw new Error(`expected only one query param but got multiple: ${key}`); + } + + if (!(key in query)) { + return null; + } + + return query[key] as T; +}; + +const queryParamParsers: { + [K in keyof StepperState]: (query: ParsedUrlQuery) => StepperState[K]; +} = { + stepperState: (query) => + getQueryParamAsString( + query, + "stepperState" + ) ?? "unstarted", + repository: (query) => ({ + namespace: getQueryParamAsString(query, "githubNamespace"), + repository: getQueryParamAsString(query, "githubRepository"), + }), + importTaskId: (query) => getQueryParamAsString(query, "importTaskId"), + importError: (query) => getQueryParamAsString(query, "importError"), + exportError: (query) => getQueryParamAsString(query, "exportError"), + splitgraphNamespace: (query) => + getQueryParamAsString(query, "splitgraphNamespace"), + splitgraphRepository: (query) => + getQueryParamAsString(query, "splitgraphRepository"), +}; + +const requireKeys = >( + obj: T, + requiredKeys: (keyof T)[] +) => { + const missingKeys = requiredKeys.filter( + (requiredKey) => !(requiredKey in obj) + ); + + if (missingKeys.length > 0) { + throw new Error("missing required keys: " + missingKeys.join(", ")); + } +}; + +const stepperStateValidators: { + [K in StepperState["stepperState"]]: (stateFromQuery: StepperState) => void; +} = { + uninitialized: () => {}, + unstarted: () => {}, + awaiting_import: (stateFromQuery) => + requireKeys(stateFromQuery, [ + "repository", + "importTaskId", + "splitgraphNamespace", + "splitgraphRepository", + ]), + import_complete: (stateFromQuery) => + requireKeys(stateFromQuery, [ + "repository", + "splitgraphNamespace", + "splitgraphRepository", + ]), + awaiting_export: (stateFromQuery) => + requireKeys(stateFromQuery, [ + "repository", + "splitgraphNamespace", + "splitgraphRepository", + ]), + export_complete: (stateFromQuery) => + requireKeys(stateFromQuery, [ + "repository", + "splitgraphNamespace", + "splitgraphRepository", + ]), +}; + +const parseStateFromRouter = (router: NextRouter): StepperState => { + const { query } = router; + + const stepperState = queryParamParsers.stepperState(query); + + const stepper = { + stepperState: stepperState, + repository: queryParamParsers.repository(query), + importTaskId: queryParamParsers.importTaskId(query), + importError: queryParamParsers.importError(query), + exportError: queryParamParsers.exportError(query), + splitgraphNamespace: queryParamParsers.splitgraphNamespace(query), + splitgraphRepository: queryParamParsers.splitgraphRepository(query), + }; + + void stepperStateValidators[stepperState](stepper); + + return stepper; +}; + +const serializeStateToQueryParams = (stepper: StepperState) => { + return JSON.parse( + JSON.stringify({ + stepperState: stepper.stepperState, + githubNamespace: stepper.repository?.namespace ?? undefined, + githubRepository: stepper.repository?.repository ?? undefined, + importTaskId: stepper.importTaskId ?? undefined, + importError: stepper.importError ?? undefined, + exportError: stepper.exportError ?? undefined, + splitgraphNamespace: stepper.splitgraphNamespace ?? undefined, + splitgraphRepository: stepper.splitgraphRepository ?? undefined, + }) + ); +}; + +const stepperReducer = ( state: StepperState, action: StepperAction ): StepperState => { @@ -142,7 +271,66 @@ export const stepperReducer = ( case "reset": return initialState; + case "initialize_from_url": + return { + ...state, + ...action.parsedFromUrl, + }; + default: return state; } }; + +const urlNeedsChange = (state: StepperState, router: NextRouter) => { + const parsedFromUrl = parseStateFromRouter(router); + + return ( + state.stepperState !== parsedFromUrl.stepperState || + state.repository?.namespace !== parsedFromUrl.repository?.namespace || + state.repository?.repository !== parsedFromUrl.repository?.repository || + state.importTaskId !== parsedFromUrl.importTaskId || + state.splitgraphNamespace !== parsedFromUrl.splitgraphNamespace || + state.splitgraphRepository !== parsedFromUrl.splitgraphRepository + ); +}; + +export const useStepperReducer = () => { + const router = useRouter(); + const [state, dispatch] = useReducer(stepperReducer, { + ...initialState, + stepperState: "uninitialized", + }); + + useEffect(() => { + dispatch({ + type: "initialize_from_url", + parsedFromUrl: parseStateFromRouter(router), + }); + }, [router.query]); + + useEffect(() => { + if (!urlNeedsChange(state, router)) { + return; + } + + if (state.stepperState === "uninitialized") { + return; + } + + console.log("push", { + pathname: router.pathname, + query: serializeStateToQueryParams(state), + }); + router.push( + { + pathname: router.pathname, + query: serializeStateToQueryParams(state), + }, + undefined, + { shallow: true } + ); + }, [state.stepperState]); + + return [state, dispatch] as const; +}; From 9e47b5ad014e62e7346e0493d166454ba150d6c9 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Sat, 27 May 2023 00:19:57 +0100 Subject: [PATCH 10/36] Move `lib/config.ts` -> `lib/config/github-tables.ts` --- .../components/ImportExportStepper/ExportPanel.tsx | 2 +- .../lib/{config.ts => config/github-tables.ts} | 0 .../pages/api/start-import-from-github.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename examples/nextjs-import-airbyte-github-export-seafowl/lib/{config.ts => config/github-tables.ts} (100%) diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx index ceb29f6..6ae7ebd 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx @@ -4,7 +4,7 @@ import { useStepper } from "./StepperContext"; import styles from "./ExportPanel.module.css"; import { ExportLoadingBars } from "./ExportLoadingBars"; -import { relevantGitHubTableNames } from "../../lib/config"; +import { relevantGitHubTableNames } from "../../lib/config/github-tables"; export const ExportPanel = () => { const [ diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts similarity index 100% rename from examples/nextjs-import-airbyte-github-export-seafowl/lib/config.ts rename to examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts index dc78648..506c529 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts @@ -3,7 +3,7 @@ import { makeAuthenticatedSplitgraphDb, claimsFromJWT, } from "../../lib/backend/splitgraph-db"; -import { relevantGitHubTableNames } from "../../lib/config"; +import { relevantGitHubTableNames } from "../../lib/config/github-tables"; const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET; From d60cd4ddba0e883a1ce0955155b70e9d01540961 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Sat, 27 May 2023 00:59:17 +0100 Subject: [PATCH 11/36] Export analytics queries to Seafowl in addition to tables Export queries to tables `monthly_user_stats` and `monthly_issue_stats` in the same schema/namespace as the tables. We also export the tables, or at least the few that we explicitly asked to import. --- .../ImportExportStepper/ExportLoadingBars.tsx | 18 ++- .../ImportExportStepper/ExportPanel.tsx | 78 ++++++++--- .../ExportTableLoadingBar.tsx | 33 +++-- .../ImportExportStepper/stepper-states.ts | 21 ++- .../lib/config/github-tables.ts | 6 +- .../lib/config/queries-to-export.ts | 116 ++++++++++++++++ .../pages/api/start-export-to-seafowl.ts | 127 +++++++++++++++--- .../pages/api/start-import-from-github.ts | 4 +- 8 files changed, 338 insertions(+), 65 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx index f830456..076cded 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportLoadingBars.tsx @@ -7,13 +7,17 @@ export const ExportLoadingBars = () => { return (
- {Array.from(exportedTablesLoading).map(({ tableName, taskId }) => ( - - ))} + {Array.from(exportedTablesLoading).map( + ({ destinationSchema, destinationTable, sourceQuery, taskId }) => ( + + ) + )}
); }; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx index 6ae7ebd..ad360be 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx @@ -4,7 +4,15 @@ import { useStepper } from "./StepperContext"; import styles from "./ExportPanel.module.css"; import { ExportLoadingBars } from "./ExportLoadingBars"; -import { relevantGitHubTableNames } from "../../lib/config/github-tables"; +import { relevantGitHubTableNamesForImport } from "../../lib/config/github-tables"; +import { makeQueriesToExport } from "../../lib/config/queries-to-export"; +import type { + ExportQueryInput, + ExportTableInput, + StartExportToSeafowlRequestShape, + StartExportToSeafowlResponseData, +} from "../../pages/api/start-export-to-seafowl"; +import { useMemo, useCallback } from "react"; export const ExportPanel = () => { const [ @@ -12,40 +20,76 @@ export const ExportPanel = () => { dispatch, ] = useStepper(); - const handleStartExport = async () => { + const queriesToExport = useMemo( + () => + makeQueriesToExport({ + splitgraphSourceRepository: splitgraphRepository, + splitgraphSourceNamespace: splitgraphNamespace, + seafowlDestinationSchema: `${splitgraphNamespace}/${splitgraphRepository}`, + }), + [splitgraphRepository, splitgraphNamespace] + ); + + const tablesToExport = useMemo( + () => + relevantGitHubTableNamesForImport.map((tableName) => ({ + namespace: splitgraphNamespace, + repository: splitgraphRepository, + table: tableName, + })), + [ + splitgraphNamespace, + splitgraphRepository, + relevantGitHubTableNamesForImport, + ] + ); + + const handleStartExport = useCallback(async () => { try { const response = await fetch("/api/start-export-to-seafowl", { method: "POST", body: JSON.stringify({ - tables: relevantGitHubTableNames.map((tableName) => ({ - namespace: splitgraphNamespace, - repository: splitgraphRepository, - table: tableName, - })), - }), + tables: tablesToExport, + queries: queriesToExport, + } as StartExportToSeafowlRequestShape), headers: { "Content-Type": "application/json", }, }); - const data = await response.json(); + const data = (await response.json()) as StartExportToSeafowlResponseData; + + if ("error" in data && data["error"]) { + throw new Error(data["error"]); + } - if (!data.tables || !data.tables.length) { + if (!("tables" in data) || !("queries" in data)) { throw new Error("Response missing tables"); } dispatch({ type: "start_export", - tables: data.tables.map( - ({ tableName, taskId }: { tableName: string; taskId: string }) => ({ - taskId, - tableName, - }) - ), + tables: [ + ...data["queries"].map( + ({ sourceQuery, taskId, destinationSchema, destinationTable }) => ({ + taskId, + destinationTable, + destinationSchema, + sourceQuery, + }) + ), + ...data["tables"].map( + ({ destinationTable, destinationSchema, taskId }) => ({ + taskId, + destinationTable, + destinationSchema, + }) + ), + ], }); } catch (error) { dispatch({ type: "export_error", error: error.message }); } - }; + }, [queriesToExport, tablesToExport, dispatch]); return (
diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx index 7551b58..945fbd3 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportTableLoadingBar.tsx @@ -3,22 +3,28 @@ import { useStepper } from "./StepperContext"; import styles from "./ExportTableLoadingBar.module.css"; interface ExportTableLoadingBarProps { - tableName: string; + destinationTable: string; + destinationSchema: string; + sourceQuery?: string; taskId: string; } export const ExportTableLoadingBar = ({ - tableName, + destinationTable, + destinationSchema, + sourceQuery, taskId, }: React.PropsWithoutRef) => { const [{ stepperState, exportedTablesLoading }, dispatch] = useStepper(); useEffect(() => { - if (!taskId || !tableName) { - console.log("Don't check export until we have taskId and tableName"); + if (!taskId || !destinationTable) { + console.log( + "Don't check export until we have taskId and destinationTable" + ); console.table({ taskId, - tableName, + destinationTable, }); return; } @@ -44,7 +50,12 @@ export const ExportTableLoadingBar = ({ if (data.completed) { dispatch({ type: "export_table_task_complete", - completedTable: { tableName, taskId }, + completedTable: { + destinationTable, + taskId, + destinationSchema, + sourceQuery, + }, }); } else if (data.error) { if (!data.completed) { @@ -56,14 +67,14 @@ export const ExportTableLoadingBar = ({ } catch (error) { dispatch({ type: "export_error", - error: `Error exporting ${tableName}: ${error.message}`, + error: `Error exporting ${destinationTable}: ${error.message}`, }); } }; const interval = setInterval(pollExportTask, 3000); return () => clearInterval(interval); - }, [stepperState, tableName, taskId, dispatch]); + }, [stepperState, destinationTable, taskId, dispatch]); const isLoading = !!Array.from(exportedTablesLoading).find( (t) => t.taskId === taskId @@ -73,10 +84,10 @@ export const ExportTableLoadingBar = ({
{isLoading - ? `Loading ${tableName}...` - : `Successfully exported ${tableName}`} + ? `Loading ${destinationTable}...` + : `Successfully exported ${destinationTable}`}
-
{tableName}
+
{destinationTable}
); }; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts index c8fdaf1..16ca39a 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts @@ -3,7 +3,12 @@ import { ParsedUrlQuery } from "querystring"; import { useEffect, useReducer } from "react"; export type GitHubRepository = { namespace: string; repository: string }; -type ExportTable = { tableName: string; taskId: string }; +type ExportTable = { + destinationSchema: string; + destinationTable: string; + taskId: string; + sourceQuery?: string; +}; export type StepperState = { stepperState: @@ -210,8 +215,18 @@ const stepperReducer = ( const exportedTablesLoading = new Set(); const exportedTablesCompleted = new Set(); - for (const { tableName, taskId } of tables) { - exportedTablesLoading.add({ tableName, taskId }); + for (const { + destinationTable, + destinationSchema, + sourceQuery, + taskId, + } of tables) { + exportedTablesLoading.add({ + destinationTable, + destinationSchema, + sourceQuery, + taskId, + }); } return { diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts index 8f6719a..92e3903 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/github-tables.ts @@ -7,7 +7,7 @@ * Note that Airbyte will still import tables that depend on these tables due * to foreign keys, and will also import airbyte metaata tables. */ -export const relevantGitHubTableNames = `commits +export const relevantGitHubTableNamesForImport = `commits comments pull_requests pull_request_stats @@ -17,12 +17,12 @@ issue_reactions` /** * List of "downstream" GitHub table names that will be imported by default by - * the `airbyte-github` connector, given the list of `relevantGitHubTableNames`, + * the `airbyte-github` connector, given the list of `relevantGitHubTableNamesForImport`, * because they're either an Airbyte meta table or a table that depends on * one of the "relevant" tables. * * This is manually curated and might not be totally accurate. It's up to date - * given the following list of `relevantGitHubTableNames`: + * given the following list of `relevantGitHubTableNamesForImport`: * * ``` * commits diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts new file mode 100644 index 0000000..099d4f2 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/config/queries-to-export.ts @@ -0,0 +1,116 @@ +/** + * Return a a list of queries to export from Splitgraph to Seafowl, given the + * source repository (where the GitHub data was imported into), and the destination + * schema (where the data will be exported to at Seafowl). + */ +export const makeQueriesToExport = ({ + splitgraphSourceRepository, + splitgraphSourceNamespace, + seafowlDestinationSchema, + splitgraphSourceImageHashOrTag = "latest", +}: { + splitgraphSourceNamespace: string; + splitgraphSourceRepository: string; + seafowlDestinationSchema: string; + splitgraphSourceImageHashOrTag?: string; +}): { + sourceQuery: string; + destinationSchema: string; + destinationTable: string; +}[] => [ + { + destinationSchema: seafowlDestinationSchema, + destinationTable: "monthly_user_stats", + sourceQuery: ` + WITH + + commits AS ( + SELECT + date_trunc('month', created_at) AS created_at_month, + author->>'login' AS username, + count(*) as no_commits + FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".commits + GROUP BY 1, 2 + ), + + comments AS ( + SELECT + date_trunc('month', created_at) AS created_at_month, + "user"->>'login' AS username, + count(*) filter (where exists(select regexp_matches(issue_url, '.*/pull/.*'))) as no_pull_request_comments, + count(*) filter (where exists(select regexp_matches(issue_url, '.*/issue/.*'))) as no_issue_comments, + sum(length(body)) as total_comment_length + FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".comments + GROUP BY 1, 2 + ), + + pull_requests AS ( + WITH pull_request_creator AS ( + SELECT id, "user"->>'login' AS username + FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_requests + ) + + SELECT + date_trunc('month', updated_at) AS created_at_month, + username, + count(*) filter (where merged = true) AS merged_pull_requests, + count(*) AS total_pull_requests, + sum(additions::integer) filter (where merged = true) AS lines_added, + sum(deletions::integer) filter (where merged = true) AS lines_deleted + FROM "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}".pull_request_stats + INNER JOIN pull_request_creator USING (id) + GROUP BY 1, 2 + ), + + all_months_users AS ( + SELECT DISTINCT created_at_month, username FROM commits + UNION SELECT DISTINCT created_at_month, username FROM comments + UNION SELECT DISTINCT created_at_month, username FROM pull_requests + ), + + user_stats AS ( + SELECT + amu.created_at_month, + amu.username, + COALESCE(cmt.no_commits, 0) AS no_commits, + COALESCE(cmnt.no_pull_request_comments, 0) AS no_pull_request_comments, + COALESCE(cmnt.no_issue_comments, 0) AS no_issue_comments, + COALESCE(cmnt.total_comment_length, 0) AS total_comment_length, + COALESCE(pr.merged_pull_requests, 0) AS merged_pull_requests, + COALESCE(pr.total_pull_requests, 0) AS total_pull_requests, + COALESCE(pr.lines_added, 0) AS lines_added, + COALESCE(pr.lines_deleted, 0) AS lines_deleted + + FROM all_months_users amu + LEFT JOIN commits cmt ON amu.created_at_month = cmt.created_at_month AND amu.username = cmt.username + LEFT JOIN comments cmnt ON amu.created_at_month = cmnt.created_at_month AND amu.username = cmnt.username + LEFT JOIN pull_requests pr ON amu.created_at_month = pr.created_at_month AND amu.username = pr.username + + ORDER BY created_at_month ASC, username ASC + ) + + SELECT * FROM user_stats; +`, + }, + { + destinationSchema: seafowlDestinationSchema, + destinationTable: "monthly_issue_stats", + sourceQuery: ` +SELECT + issue_number, + date_trunc('month', created_at::TIMESTAMP) as created_at_month, + COUNT(*) AS total_reacts, + COUNT(*) FILTER (WHERE content = '+1') AS no_plus_one, + COUNT(*) FILTER (WHERE content = '-1') AS no_minus_one, + COUNT(*) FILTER (WHERE content = 'laugh') AS no_laugh, + COUNT(*) FILTER (WHERE content = 'confused') AS no_confused, + COUNT(*) FILTER (WHERE content = 'heart') AS no_heart, + COUNT(*) FILTER (WHERE content = 'hooray') AS no_hooray, + COUNT(*) FILTER (WHERE content = 'rocket') AS no_rocket, + COUNT(*) FILTER (WHERE content = 'eyes') AS no_eyes +FROM + "${splitgraphSourceNamespace}/${splitgraphSourceRepository}:${splitgraphSourceImageHashOrTag}"."issue_reactions" +GROUP BY 1, 2 ORDER BY 2, 3 DESC; +`, + }, +]; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts index 4e95752..f616d37 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-export-to-seafowl.ts @@ -1,16 +1,40 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { makeAuthenticatedSplitgraphDb } from "../../lib/backend/splitgraph-db"; -type ResponseData = +export type StartExportToSeafowlRequestShape = + | { + tables: ExportTableInput[]; + } + | { queries: ExportQueryInput[] } + | { tables: ExportTableInput[]; queries: ExportQueryInput[] }; + +export type StartExportToSeafowlResponseData = | { tables: { - tableName: string; + destinationTable: string; + destinationSchema: string; + taskId: string; + }[]; + queries: { + sourceQuery: string; + destinationSchema: string; + destinationTable: string; taskId: string; }[]; } | { error: string }; -type TableInput = { namespace: string; repository: string; table: string }; +export type ExportTableInput = { + namespace: string; + repository: string; + table: string; +}; + +export type ExportQueryInput = { + sourceQuery: string; + destinationSchema: string; + destinationTable: string; +}; /** * To manually send a request, example: @@ -23,16 +47,22 @@ curl -i \ */ export default async function handler( req: NextApiRequest, - res: NextApiResponse + res: NextApiResponse ) { const db = makeAuthenticatedSplitgraphDb(); - const { tables } = req.body; + const { tables = [], queries = [] } = req.body; + + if (tables.length === 0 && queries.length === 0) { + res.status(400).json({ error: "no tables or queries provided for export" }); + return; + } + + const errors = []; if ( - !tables || - !tables.length || + tables.length > 0 && !tables.every( - (t: TableInput) => + (t: ExportTableInput) => t.namespace && t.repository && t.table && @@ -41,17 +71,39 @@ export default async function handler( typeof t.table === "string" ) ) { - res.status(400).json({ error: "invalid tables input in request body" }); + errors.push("invalid tables input in request body"); + } + + if ( + queries.length > 0 && + !queries.every( + (q: ExportQueryInput) => + q.sourceQuery && + q.destinationSchema && + q.destinationTable && + typeof q.sourceQuery === "string" && + typeof q.destinationSchema === "string" && + typeof q.destinationTable === "string" + ) + ) { + errors.push("invalid queries input in request body"); + } + + if (errors.length > 0) { + res.status(400).json({ error: `Invalid request: ${errors.join(", ")}` }); return; } try { - const exportingTables = await startExport({ - db, - tables, - }); + const { tables: exportingTables, queries: exportingQueries } = + await startExport({ + db, + tables, + queries, + }); res.status(200).json({ tables: exportingTables, + queries: exportingQueries, }); } catch (err) { res.status(400).json({ @@ -63,15 +115,26 @@ export default async function handler( const startExport = async ({ db, tables, + queries, }: { db: ReturnType; - tables: TableInput[]; + tables: ExportTableInput[]; + queries: ExportQueryInput[]; }) => { await db.fetchAccessToken(); const response = await db.exportData( "export-to-seafowl", { + queries: queries.map((query) => ({ + source: { + query: query.sourceQuery, + }, + destination: { + schema: query.destinationSchema, + table: query.destinationTable, + }, + })), tables: tables.map((splitgraphSource) => ({ source: { repository: splitgraphSource.repository, @@ -91,13 +154,33 @@ const startExport = async ({ throw new Error(JSON.stringify(response.error)); } - const loadingTables: { taskId: string; tableName: string }[] = - response.taskIds.tables.map( - (t: { jobId: string; sourceTable: string }) => ({ - taskId: t.jobId, - tableName: t.sourceTable, - }) - ); + const loadingTables = response.taskIds.tables.map( + (t: { jobId: string; sourceTable: string; sourceRepository: string }) => ({ + taskId: t.jobId, + destinationTable: t.sourceTable, + destinationSchema: t.sourceRepository, + }) + ); + + const loadingQueries = response.taskIds.queries.map( + ( + queryJob: { + jobId: string; + destinationSchema: string; + destinationTable: string; + sourceQuery: string; + }, + i: number + ) => ({ + taskId: queryJob.jobId, + destinationSchema: queries[i].destinationSchema, + destinationTable: queries[i].destinationTable, + sourceQuery: queries[i].sourceQuery, + }) + ); - return loadingTables; + return { + tables: loadingTables, + queries: loadingQueries, + }; }; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts index 506c529..5f7c079 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/start-import-from-github.ts @@ -3,7 +3,7 @@ import { makeAuthenticatedSplitgraphDb, claimsFromJWT, } from "../../lib/backend/splitgraph-db"; -import { relevantGitHubTableNames } from "../../lib/config/github-tables"; +import { relevantGitHubTableNamesForImport } from "../../lib/config/github-tables"; const GITHUB_PAT_SECRET = process.env.GITHUB_PAT_SECRET; @@ -101,7 +101,7 @@ const startImport = async ({ namespace: splitgraphNamespace, repository: splitgraphDestinationRepository, tables: [ - ...relevantGitHubTableNames.map((t) => ({ + ...relevantGitHubTableNamesForImport.map((t) => ({ name: t, options: {}, schema: [], From 17d8bc9eafdb0581a8f73f5abc8b7527f6a80115 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Mon, 29 May 2023 19:04:51 +0100 Subject: [PATCH 12/36] Track completed import/exports in a meta repository on Splitgraph After an import/export has completed, insert a row into the meta table, which we will also use to fetch the previously imported repositories from the client side when rendering the sidebar. We don't have transactional guarantees on the DDN, so we can't do `INSERT ON CONFLICT`, so instead we avoid duplicate rows by first selecting the existing row, and returning `204` if it's already been inserted into the `completed_repositories` table. However, I did notice that when I inserted the same row twice, it only showed up once when I made a selection in the Console. I don't know if this was due to a race condition, a bug, or because it's using the entire row as a compound primary key and for some reason requiring that it be unique. --- .../.env.test.local | 3 + .../ImportExportStepper/ExportPanel.tsx | 9 + .../ImportExportStepper/StepperContext.tsx | 2 +- .../ImportExportStepper/stepper-states.ts | 100 +++++++++-- .../env-vars.d.ts | 53 ++++++ .../lib/backend/splitgraph-db.ts | 11 +- .../pages/api/mark-import-export-complete.ts | 168 ++++++++++++++++++ 7 files changed, 326 insertions(+), 20 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local index 473bf7d..dfb13b4 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local +++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local @@ -7,6 +7,9 @@ SPLITGRAPH_API_KEY="********************************" SPLITGRAPH_API_SECRET="********************************" +# This should match the username associated with the API key +SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****" + # Create a GitHub token that can query the repositories you want to connect # For example, a token with read-only access to public repos is sufficient # CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx index ad360be..bcfd800 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/ExportPanel.tsx @@ -45,6 +45,8 @@ export const ExportPanel = () => { ); const handleStartExport = useCallback(async () => { + const abortController = new AbortController(); + try { const response = await fetch("/api/start-export-to-seafowl", { method: "POST", @@ -55,6 +57,7 @@ export const ExportPanel = () => { headers: { "Content-Type": "application/json", }, + signal: abortController.signal, }); const data = (await response.json()) as StartExportToSeafowlResponseData; @@ -87,8 +90,14 @@ export const ExportPanel = () => { ], }); } catch (error) { + if (error.name === "AbortError") { + return; + } + dispatch({ type: "export_error", error: error.message }); } + + return () => abortController.abort(); }, [queriesToExport, tablesToExport, dispatch]); return ( diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx index 0fdb7c0..f80ed7f 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/StepperContext.tsx @@ -1,5 +1,5 @@ // StepperContext.tsx -import React, { useReducer, useContext, ReactNode } from "react"; +import React, { useContext, ReactNode } from "react"; import { StepperState, StepperAction, diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts index 16ca39a..b1e5e51 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts @@ -45,11 +45,6 @@ export type StepperAction = | { type: "reset" } | { type: "initialize_from_url"; parsedFromUrl: StepperState }; -type ExtractStepperAction = Extract< - StepperAction, - { type: T } ->; - const initialState: StepperState = { stepperState: "unstarted", repository: null, @@ -62,19 +57,6 @@ const initialState: StepperState = { exportError: null, }; -// FOR DEBUGGING: uncomment for hardcoded state initialization -// export const initialState: StepperState = { -// ...normalInitialState, -// stepperState: "import_complete", -// splitgraphNamespace: "miles", -// splitgraphRepository: "import-via-nextjs", -// }; - -type ActionParams = Omit< - ExtractStepperAction, - "type" ->; - const getQueryParamAsString = ( query: ParsedUrlQuery, key: string @@ -310,6 +292,86 @@ const urlNeedsChange = (state: StepperState, router: NextRouter) => { ); }; +/** + * When the export has completed, send a request to /api/mark-import-export-complete + * which will insert the repository into the metadata table, which we query to + * render the sidebar + */ +const useMarkAsComplete = ( + state: StepperState, + dispatch: React.Dispatch +) => { + useEffect(() => { + if (state.stepperState !== "export_complete") { + return; + } + + const { + repository: { + namespace: githubSourceNamespace, + repository: githubSourceRepository, + }, + splitgraphRepository: splitgraphDestinationRepository, + } = state; + + // NOTE: Make sure to abort request so that in React 18 development mode, + // when effect runs twice, the second request is aborted and we don't have + // a race condition with two requests inserting into the table (where we have no transactional + // integrity and manually do a SELECT before the INSERT to check if the row already exists) + const abortController = new AbortController(); + + const markImportExportComplete = async () => { + try { + const response = await fetch("/api/mark-import-export-complete", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + githubSourceNamespace, + githubSourceRepository, + splitgraphDestinationRepository, + }), + signal: abortController.signal, + }); + + if (!response.ok) { + throw new Error("Failed to mark import/export as complete"); + } + + const data = await response.json(); + + if (!data.status) { + throw new Error( + "Got unexpected resposne shape when marking import/export complete" + ); + } + + if (data.error) { + throw new Error( + `Failed to mark import/export complete: ${data.error}` + ); + } + + console.log("Marked import/export as complete"); + } catch (error) { + if (error.name === "AbortError") { + return; + } + + dispatch({ + type: "export_error", + error: error.message ?? error.toString(), + }); + } + }; + + markImportExportComplete(); + + return () => abortController.abort(); + }, [state, dispatch]); +}; + export const useStepperReducer = () => { const router = useRouter(); const [state, dispatch] = useReducer(stepperReducer, { @@ -317,6 +379,8 @@ export const useStepperReducer = () => { stepperState: "uninitialized", }); + useMarkAsComplete(state, dispatch); + useEffect(() => { dispatch({ type: "initialize_from_url", diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts index 5328009..2b0d888 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/env-vars.d.ts @@ -37,5 +37,58 @@ namespace NodeJS { * This is useful for debugging and development. */ MITMPROXY_ADDRESS?: string; + + /** + * The namespace of the repository in Splitgraph where metadata is stored + * containing the state of imported GitHub repositories, which should contain + * the repository `SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY`. + * + * This should be defined in `.env.local`, since it's not checked into Git + * and can vary between users. It should match the username associated with + * the `SPLITGRAPH_API_KEY` + * + * Example: + * + * ``` + * miles/splitgraph-github-analytics.completed_repositories + * ^^^^^ + * SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE=miles + * ``` + */ + SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE: string; + + /** + * The repository (no namespace) in Splitgraph where metadata is stored + * containing the state of imported GitHub repositories, which should be a + * repository contained inside `SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE`. + * + * This is defined by default in `.env` which is checked into Git. + * + * * Example: + * + * ``` + * miles/splitgraph-github-analytics.completed_repositories + * ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=splitgraph-github-analytics + * ``` + */ + SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY: string; + + /** + * The name of the table containing completed repositories, which are inserted + * when the import/export is complete, and which can be queried to render the + * sidebar containing previously imported github repositories. + * + * This is defined by default in `.env` which is checked into Git. + * + * Example: + * + * ``` + * miles/splitgraph-github-analytics.completed_repositories + * ^^^^^^^^^^^^^^^^^^^^^^ + * SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories + * ``` + */ + SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE: string; } } diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts index 66ba751..ca1f2f2 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/lib/backend/splitgraph-db.ts @@ -1,4 +1,4 @@ -import { makeSplitgraphDb } from "@madatdata/core"; +import { makeSplitgraphDb, makeSplitgraphHTTPContext } from "@madatdata/core"; // TODO: fix plugin exports import { makeDefaultPluginList } from "@madatdata/db-splitgraph"; @@ -33,6 +33,15 @@ export const makeAuthenticatedSplitgraphDb = () => }), }); +export const makeAuthenticatedSplitgraphHTTPContext = () => + makeSplitgraphHTTPContext({ + authenticatedCredential, + plugins: makeDefaultPluginList({ + graphqlEndpoint: defaultSplitgraphHost.baseUrls.gql, + authenticatedCredential, + }), + }); + // TODO: export this utility function from the library export const claimsFromJWT = (jwt?: string) => { if (!jwt) { diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts new file mode 100644 index 0000000..c981b03 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/pages/api/mark-import-export-complete.ts @@ -0,0 +1,168 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { + makeAuthenticatedSplitgraphHTTPContext, + claimsFromJWT, +} from "../../lib/backend/splitgraph-db"; + +export type MarkImportExportCompleteRequestShape = { + githubSourceNamespace: string; + githubSourceRepository: string; + splitgraphDestinationRepository: string; +}; + +export type MarkImportExportCompleteSuccessResponse = { + status: "inserted"; +}; + +export type MarkImportExportCompleteResponseData = + | MarkImportExportCompleteSuccessResponse + | { error: string }; + +const META_NAMESPACE = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE; +const META_REPOSITORY = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY; +const META_TABLE = process.env.SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE; + +/** + * To manually send a request, example: + +```bash +curl -i \ + -H "Content-Type: application/json" http://localhost:3000/api/mark-import-export-complete \ + -d@- < +) { + if (!META_NAMESPACE) { + res.status(400).json({ + error: + "Missing env var: SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE " + + "Is it in .env.local or Vercel secrets?", + }); + return; + } + + if (!META_REPOSITORY) { + res.status(400).json({ + error: + "Missing env var: SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY " + + "Is it in .env or Vercel environment variables?", + }); + return; + } + + const missingOrInvalidKeys = [ + "githubSourceNamespace", + "githubSourceRepository", + "splitgraphDestinationRepository", + ].filter( + (requiredKey) => + !(requiredKey in req.body) || + typeof req.body[requiredKey] !== "string" || + !req.body[requiredKey] || + !isSQLSafe(req.body[requiredKey]) + ); + + if (missingOrInvalidKeys.length > 0) { + res.status(400).json({ + error: `missing, non-string, empty or invalid keys: ${missingOrInvalidKeys.join( + ", " + )}`, + }); + return; + } + + try { + const { status } = await markImportExportAsComplete({ + githubSourceNamespace: req.body.githubSourceNamespace, + githubSourceRepository: req.body.githubSourceRepository, + splitgraphDestinationRepository: req.body.splitgraphDestinationRepository, + }); + + if (status === "already exists") { + res.status(204).end(); + return; + } + + res.status(200).json({ status }); + return; + } catch (err) { + res.status(400).json({ error: err }); + return; + } +} + +/** + * NOTE: We assume that this table already exists. If it does not exist, you can + * create it manually with a query like this in https://www.splitgraph.com/query : + * + * ```sql +CREATE TABLE IF NOT EXISTS "miles/github-analytics-metadata".completed_repositories ( + github_namespace VARCHAR NOT NULL, + github_repository VARCHAR NOT NULL, + splitgraph_namespace VARCHAR NOT NULL, + splitgraph_repository VARCHAR NOT NULL, + completed_at TIMESTAMP NOT NULL +); +``` + */ +const markImportExportAsComplete = async ({ + splitgraphDestinationRepository, + githubSourceNamespace, + githubSourceRepository, +}: MarkImportExportCompleteRequestShape): Promise<{ + status: "already exists" | "inserted"; +}> => { + const { db, client } = makeAuthenticatedSplitgraphHTTPContext(); + const { username } = claimsFromJWT((await db.fetchAccessToken()).token); + + // NOTE: We also assume that META_NAMESPACE is the same as destination namespace + if (!username || username !== META_NAMESPACE) { + throw new Error( + "Authenticated user does not match SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE" + ); + } + + // We don't want to insert the row if it already exists + // Note that Splitgraph doesn't support constraints so we can't use INSERT ON CONFLICT + + const existingRows = await client.execute(` + SELECT splitgraph_repository FROM "${META_NAMESPACE}/${META_REPOSITORY}"."${META_TABLE}" + WHERE github_namespace = '${githubSourceNamespace}' + AND github_repository = '${githubSourceRepository}' + AND splitgraph_namespace = '${META_NAMESPACE}' + AND splitgraph_repository = '${splitgraphDestinationRepository}'; + `); + + if (existingRows.response && existingRows.response.rows.length > 0) { + return { status: "already exists" }; + } + + await client.execute(`INSERT INTO "${META_NAMESPACE}/${META_REPOSITORY}"."${META_TABLE}" ( + github_namespace, + github_repository, + splitgraph_namespace, + splitgraph_repository, + completed_at +) VALUES ( + '${githubSourceNamespace}', + '${githubSourceRepository}', + '${META_NAMESPACE}', + '${splitgraphDestinationRepository}', + NOW() +);`); + + return { status: "inserted" }; +}; + +/** + * Return `false` if the string contains any character other than alphanumeric, + * `-`, `_`, or `.` + */ +const isSQLSafe = (str: string) => !/[^a-z0-9\-_\.]/.test(str); From 3ba9e97c30be6a1d0ab6d6cdc72054746d8aee0e Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Mon, 29 May 2023 22:21:19 +0100 Subject: [PATCH 13/36] Support ?debug=1 parameter in URL of stepper to render DebugPane --- .../components/ImportExportStepper/Stepper.tsx | 15 ++++++++++++--- .../ImportExportStepper/stepper-states.ts | 14 ++++++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx index 93f987f..101ad5f 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/Stepper.tsx @@ -4,12 +4,22 @@ import { ImportPanel } from "./ImportPanel"; import { ExportPanel } from "./ExportPanel"; import styles from "./Stepper.module.css"; +import { useRouter } from "next/router"; const StepperOrLoading = ({ children }: { children: React.ReactNode }) => { - const [{ stepperState }] = useStepper(); + const [{ stepperState, debug }] = useStepper(); return ( - <>{stepperState === "uninitialized" ?
........
: children} + <> + {stepperState === "uninitialized" ? ( +
........
+ ) : ( + <> + {debug && } + {children} + + )} + ); }; @@ -18,7 +28,6 @@ export const Stepper = () => {
- diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts index b1e5e51..5895bdd 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/ImportExportStepper/stepper-states.ts @@ -26,6 +26,7 @@ export type StepperState = { exportedTablesLoading?: Set; exportedTablesCompleted?: Set; exportError?: string; + debug?: string | null; }; export type StepperAction = @@ -55,6 +56,7 @@ const initialState: StepperState = { exportedTablesCompleted: new Set(), importError: null, exportError: null, + debug: null, }; const getQueryParamAsString = ( @@ -91,6 +93,7 @@ const queryParamParsers: { getQueryParamAsString(query, "splitgraphNamespace"), splitgraphRepository: (query) => getQueryParamAsString(query, "splitgraphRepository"), + debug: (query) => getQueryParamAsString(query, "debug"), }; const requireKeys = >( @@ -151,6 +154,7 @@ const parseStateFromRouter = (router: NextRouter): StepperState => { exportError: queryParamParsers.exportError(query), splitgraphNamespace: queryParamParsers.splitgraphNamespace(query), splitgraphRepository: queryParamParsers.splitgraphRepository(query), + debug: queryParamParsers.debug(query), }; void stepperStateValidators[stepperState](stepper); @@ -169,6 +173,7 @@ const serializeStateToQueryParams = (stepper: StepperState) => { exportError: stepper.exportError ?? undefined, splitgraphNamespace: stepper.splitgraphNamespace ?? undefined, splitgraphRepository: stepper.splitgraphRepository ?? undefined, + debug: stepper.debug ?? undefined, }) ); }; @@ -339,6 +344,11 @@ const useMarkAsComplete = ( throw new Error("Failed to mark import/export as complete"); } + if (response.status === 204) { + console.log("Repository already exists in metadata table"); + return; + } + const data = await response.json(); if (!data.status) { @@ -397,10 +407,6 @@ export const useStepperReducer = () => { return; } - console.log("push", { - pathname: router.pathname, - query: serializeStateToQueryParams(state), - }); router.push( { pathname: router.pathname, From e6c592ed496dcc97adc9c08369696c780a6fd380 Mon Sep 17 00:00:00 2001 From: Miles Richardson Date: Mon, 29 May 2023 22:22:28 +0100 Subject: [PATCH 14/36] Implement sidebar and stub out page for imported repository to show the charts The sidebar queries the DDN from the client-side with `useSql` from `@madatdata/react`, using the default anonymous (thus read-only) credential to query the "metadata table" that includes the list of repositories that have had a succesful import, and it links to a page for each one, which is currently a stub but where we will show the chart(s) with Observable Plot. --- .../.env | 7 ++ .../.env.test.local | 2 +- .../RepositoryAnalytics/Charts.module.css | 3 + .../components/RepositoryAnalytics/Charts.tsx | 31 ++++++++ .../components/Sidebar.tsx | 73 ++++++++++++++++--- .../env-vars.d.ts | 16 ++-- .../[github_repository].tsx | 39 ++++++++++ .../pages/_app.tsx | 14 +++- .../pages/api/mark-import-export-complete.ts | 15 ++-- .../pages/index.tsx | 51 ++----------- .../types.ts | 6 ++ 11 files changed, 184 insertions(+), 73 deletions(-) create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/.env create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/pages/[github_namespace]/[github_repository].tsx create mode 100644 examples/nextjs-import-airbyte-github-export-seafowl/types.ts diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env b/examples/nextjs-import-airbyte-github-export-seafowl/.env new file mode 100644 index 0000000..9ac0d11 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env @@ -0,0 +1,7 @@ +# This file contains public environment variables and is therefore checked into the repo +# For secret environment variables, see `.env.local` which is _not_ checked into the repo +# Read env-vars.d.ts for expected variable names +# See more: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables + +NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=github-analytics-metadata +NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local index dfb13b4..2d80ce0 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local +++ b/examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local @@ -8,7 +8,7 @@ SPLITGRAPH_API_KEY="********************************" SPLITGRAPH_API_SECRET="********************************" # This should match the username associated with the API key -SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****" +NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****" # Create a GitHub token that can query the repositories you want to connect # For example, a token with read-only access to public repos is sufficient diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css new file mode 100644 index 0000000..2142170 --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.module.css @@ -0,0 +1,3 @@ +.charts { + background: inherit; +} diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx new file mode 100644 index 0000000..d5b3b3f --- /dev/null +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/RepositoryAnalytics/Charts.tsx @@ -0,0 +1,31 @@ +import style from "./Charts.module.css"; + +import type { ImportedRepository } from "../../types"; + +export interface ChartsProps { + importedRepository: ImportedRepository; +} + +export const Charts = ({ + importedRepository: { + githubNamespace, + githubRepository, + splitgraphNamespace, + splitgraphRepository, + }, +}: ChartsProps) => { + return ( + + ); +}; diff --git a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx index f3ccb2f..c7574cb 100644 --- a/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx +++ b/examples/nextjs-import-airbyte-github-export-seafowl/components/Sidebar.tsx @@ -1,29 +1,78 @@ -import React from "react"; +import React, { useMemo } from "react"; import Link from "next/link"; import styles from "./Sidebar.module.css"; +import { useSql } from "@madatdata/react"; -export interface GitHubRepository { - namespace: string; - repository: string; -} +import type { ImportedRepository } from "../types"; -interface SidebarProps { - repositories: GitHubRepository[]; -} +const META_REPOSITORY = + process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY; +const META_NAMESPACE = + process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE; +const META_TABLE = + process.env.NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE; + +const useImportedRepositories = (): ImportedRepository[] => { + const { response, error } = useSql<{ + githubNamespace: string; + githubRepository: string; + splitgraphNamespace: string; + splitgraphRepository: string; + }>( + ` + WITH ordered_repos AS ( + SELECT + github_namespace, + github_repository, + splitgraph_namespace, + splitgraph_repository, + completed_at + FROM "${META_NAMESPACE}/${META_REPOSITORY}"."${META_TABLE}" + ORDER BY completed_at DESC + ) + SELECT DISTINCT + github_namespace AS "githubNamespace", + github_repository AS "githubRepository", + splitgraph_namespace AS "splitgraphNamespace", + splitgraph_repository AS "splitgraphRepository" + FROM ordered_repos; + ` + ); + + const repositories = useMemo(() => { + if (error) { + console.warn("Error fetching repositories:", error); + return []; + } + + if (!response) { + console.warn("No response received"); + return []; + } + + return response.rows ?? []; + }, [error, response]); + + return repositories; +}; + +export const Sidebar = () => { + const repositories = useImportedRepositories(); -export const Sidebar = ({ repositories }: React.PropsWithRef) => { return (
); }; -const SplitgraphRepoLink = ({ +export const SplitgraphRepoLink = ({ splitgraphNamespace, splitgraphRepository, -}: ImportedRepository) => { +}: Pick< + ImportedRepository, + "splitgraphNamespace" | "splitgraphRepository" +>) => { return ( { +}: Pick) => { return ( string; + importedRepository: SplitgraphRepository; + makeQuery: (repo: SplitgraphRepository) => string; tableName: string; }) => { return ( @@ -84,18 +98,42 @@ const SplitgraphQueryLink = ({ href={makeSplitgraphQueryHref(makeQuery(importedRepository))} target="_blank" > - Query {tableName} in the Splitgraph Console + Query the {tableName} table in the Splitgraph Console ); }; +export const SplitgraphStargazersQueryLink = ({ + ...importedRepository +}: SplitgraphRepository) => { + return ( + + ); +}; + +export const SeafowlStargazersQueryLink = ({ + ...importedRepository +}: SplitgraphRepository) => { + return ( + + ); +}; + const SeafowlQueryLink = ({ importedRepository, makeQuery, tableName, }: { - importedRepository: ImportedRepository; - makeQuery: (repo: ImportedRepository) => string; + importedRepository: SplitgraphRepository; + makeQuery: (repo: SplitgraphRepository) => string; tableName: string; }) => { return ( @@ -127,3 +165,58 @@ const makeSeafowlQueryHref = (sqlQuery: string) => { "database-name": META_NAMESPACE, })}`; }; + +export const SplitgraphEmbeddedQuery = ({ + importedRepository, + makeQuery, +}: { + importedRepository: SplitgraphRepository; + makeQuery: (repo: SplitgraphRepository) => string; + tableName: string; +}) => { + return ( +