diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7ec2448..373eaf4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -87,29 +87,24 @@ jobs: app_id: ${{ secrets.GH_APP_ID }} private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} - # We want to redeploy, whenever the tools.json file changes - # so make sure to hash the file and use it as a tag for the Docker image - - name: 'Download tools.json File' - run: curl -sL https://github.com/analysis-tools-dev/static-analysis/raw/master/data/api/tools.json -o ./tools.json - - - name: 'Generate Hash of tools.json File' - id: tools_json_hash - run: echo "tools_json_hash=$(sha256sum tools.json | cut -c1-7)" >> $GITHUB_ENV + # Hash the generated tools.json from the build (created by npm run build-data) + # to ensure we redeploy when tools data changes + - name: 'Generate Hash of built tools data' + run: | + echo "tools_hash=$(sha256sum data/tools.json | cut -c1-7)" >> $GITHUB_ENV - # Also take screenshots.json into account, which is at - # https://github.com/analysis-tools-dev/assets/blob/master/screenshots.json + # Also take screenshots.json into account for cache busting - name: 'Download screenshots.json File' run: curl -sL https://github.com/analysis-tools-dev/assets/raw/master/screenshots.json -o ./screenshots.json - name: 'Generate Hash of screenshots.json File' - id: screenshots_json_hash - run: echo "screenshots_json_hash=$(sha256sum screenshots.json | cut -c1-7)" >> $GITHUB_ENV + run: echo "screenshots_hash=$(sha256sum screenshots.json | cut -c1-7)" >> $GITHUB_ENV - # Image hash is a combination of the hashes + # Image hash is a combination of commit + data hashes - name: 'Set IMAGE_NAME hash' run: | short_hash=$(echo "${{ github.sha }}" | cut -c1-7) - echo "IMAGE_NAME=${{ env.IMAGE_NAME }}:$short_hash-${{ env.tools_json_hash }}-${{ env.screenshots_json_hash }}" >> $GITHUB_ENV + echo "IMAGE_NAME=${{ env.IMAGE_NAME }}:$short_hash-${{ env.tools_hash }}-${{ env.screenshots_hash }}" >> $GITHUB_ENV - name: 'Build Docker Image' env: diff --git a/.gitignore b/.gitignore index 3173cbb..c4b8f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,5 +46,12 @@ algolia-index.js credentials.json +# Generated data files (created by npm run build-data) +/data/tools.json +/data/tags.json +/data/tool-stats.json +/data/tag-stats.json .vscode + +TODO.md \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9a69ebe..6b10381 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,12 +11,10 @@ ARG PROJECT_ID COPY . /src -# Download tools.json directly in Dockerfile and detect changes -# This is done to ensure that we redeploy the app whenever the tools.json changes -ADD https://raw.githubusercontent.com/analysis-tools-dev/static-analysis/master/data/api/tools.json /src/data/api/tools.json - +# Build runs npm run build-data (prebuild hook) which fetches tools data +# from GitHub repos and generates static JSON files, then runs next build RUN npm run build -RUN rm /src/credentials.json /src/data/api/tools.json +RUN rm /src/credentials.json FROM node:20 WORKDIR /src diff --git a/lib/repositories/StatsRepository.ts b/lib/repositories/StatsRepository.ts new file mode 100644 index 0000000..be6cc63 --- /dev/null +++ b/lib/repositories/StatsRepository.ts @@ -0,0 +1,185 @@ +/** + * StatsRepository + * + * Repository class for accessing tool and tag statistics from static JSON files. + * Data is pre-built at build time by scripts/build-data.ts. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { StatsApiData, VotesApiData } from 'utils/types'; +import type { Tool, ToolsByLanguage } from '@components/tools/types'; +import { ToolsRepository } from './ToolsRepository'; +import { sortByVote } from 'utils/votes'; + +export class StatsRepository { + private static instance: StatsRepository | null = null; + private toolStatsData: StatsApiData | null = null; + private tagStatsData: StatsApiData | null = null; + + private readonly toolStatsPath: string; + private readonly tagStatsPath: string; + + private constructor() { + const dataDir = path.join(process.cwd(), 'data'); + this.toolStatsPath = path.join(dataDir, 'tool-stats.json'); + this.tagStatsPath = path.join(dataDir, 'tag-stats.json'); + } + + static getInstance(): StatsRepository { + if (!StatsRepository.instance) { + StatsRepository.instance = new StatsRepository(); + } + return StatsRepository.instance; + } + + private loadToolStats(): StatsApiData { + if (this.toolStatsData) { + return this.toolStatsData; + } + + if (!fs.existsSync(this.toolStatsPath)) { + console.warn( + 'Static tool stats not found. Run `npm run build-data` first.', + ); + return {}; + } + + const content = fs.readFileSync(this.toolStatsPath, 'utf-8'); + this.toolStatsData = JSON.parse(content) as StatsApiData; + return this.toolStatsData; + } + + private loadTagStats(): StatsApiData { + if (this.tagStatsData) { + return this.tagStatsData; + } + + if (!fs.existsSync(this.tagStatsPath)) { + console.warn( + 'Static tag stats not found. Run `npm run build-data` first.', + ); + return {}; + } + + const content = fs.readFileSync(this.tagStatsPath, 'utf-8'); + this.tagStatsData = JSON.parse(content) as StatsApiData; + return this.tagStatsData; + } + + getToolStats(): StatsApiData { + return this.loadToolStats(); + } + + getTagStats(): StatsApiData { + return this.loadTagStats(); + } + + getToolViewCount(toolId: string): number { + const stats = this.loadToolStats(); + return stats[toolId] || 0; + } + + getTagViewCount(tag: string): number { + const stats = this.loadTagStats(); + return stats[tag] || 0; + } + + getLanguageStats(): ToolsByLanguage { + const tagStats = this.loadTagStats(); + + return Object.entries(tagStats) + .sort(([, a], [, b]) => b - a) + .reduce( + (result, [key, value]) => ({ + ...result, + [key]: { + views: value, + formatters: [], + linters: [], + }, + }), + {} as ToolsByLanguage, + ); + } + + getPopularLanguageStats(votes?: VotesApiData | null): ToolsByLanguage { + const toolsRepo = ToolsRepository.getInstance(); + const tools = toolsRepo.withVotes(votes || null); + const languageStats = this.getLanguageStats(); + + for (const [toolId, tool] of Object.entries(tools)) { + const isSingleLanguage = tool.languages.length <= 2; + + if (isSingleLanguage && tool.languages.length > 0) { + const language = tool.languages[0]; + + if (languageStats[language]) { + const toolObj: Tool = { + id: toolId, + ...tool, + votes: tool.votes || 0, + } as Tool; + + if (tool.categories.includes('formatter')) { + languageStats[language].formatters.push(toolObj); + } + if (tool.categories.includes('linter')) { + languageStats[language].linters.push(toolObj); + } + + languageStats[language].formatters.sort(sortByVote); + languageStats[language].linters.sort(sortByVote); + + if (languageStats[language].formatters.length > 3) { + languageStats[language].formatters.pop(); + } + if (languageStats[language].linters.length > 3) { + languageStats[language].linters.pop(); + } + } + } + } + + // Filter out languages with no tools + for (const language of Object.keys(languageStats)) { + if ( + languageStats[language].formatters.length === 0 && + languageStats[language].linters.length === 0 + ) { + delete languageStats[language]; + } + } + + return languageStats; + } + + getMostViewedTools(votes?: VotesApiData | null): Tool[] { + const toolsRepo = ToolsRepository.getInstance(); + const tools = toolsRepo.withVotes(votes || null); + const toolStats = this.loadToolStats(); + + const mostViewedToolIds = Object.keys(toolStats); + + return mostViewedToolIds + .map((id) => { + const tool = tools[id]; + if (!tool) { + return null; + } + + return { + id, + ...tool, + votes: tool.votes || 0, + views: toolStats[id], + } as Tool & { views: number }; + }) + .filter((tool): tool is Tool & { views: number } => tool !== null); + } + + clearCache(): void { + this.toolStatsData = null; + this.tagStatsData = null; + } +} diff --git a/lib/repositories/TagsRepository.ts b/lib/repositories/TagsRepository.ts new file mode 100644 index 0000000..4424cd2 --- /dev/null +++ b/lib/repositories/TagsRepository.ts @@ -0,0 +1,171 @@ +/** + * TagsRepository + * + * Repository class for accessing tags (languages and others) from static JSON files. + * Data is pre-built at build time by scripts/build-data.ts. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { ApiTag, LanguageData, TagsType } from 'utils/types'; +import { isLanguageData } from 'utils/type-guards'; + +interface StaticTagsData { + languages: string[]; + others: string[]; +} + +export class TagsRepository { + private static instance: TagsRepository | null = null; + private tagsData: StaticTagsData | null = null; + private descriptionsData: Record | null = null; + private relatedTagsData: string[][] | null = null; + + private readonly tagsPath: string; + private readonly descriptionsPath: string; + private readonly relatedTagsPath: string; + + private constructor() { + const dataDir = path.join(process.cwd(), 'data'); + this.tagsPath = path.join(dataDir, 'tags.json'); + this.descriptionsPath = path.join(dataDir, 'descriptions.json'); + this.relatedTagsPath = path.join(dataDir, 'relatedTags.json'); + } + + static getInstance(): TagsRepository { + if (!TagsRepository.instance) { + TagsRepository.instance = new TagsRepository(); + } + return TagsRepository.instance; + } + + private loadTags(): StaticTagsData { + if (this.tagsData) { + return this.tagsData; + } + + if (!fs.existsSync(this.tagsPath)) { + console.warn( + 'Static tags data not found. Run `npm run build-data` first.', + ); + return { languages: [], others: [] }; + } + + const content = fs.readFileSync(this.tagsPath, 'utf-8'); + this.tagsData = JSON.parse(content) as StaticTagsData; + return this.tagsData; + } + + private loadDescriptions(): Record { + if (this.descriptionsData) { + return this.descriptionsData; + } + + if (!fs.existsSync(this.descriptionsPath)) { + return {}; + } + + const content = fs.readFileSync(this.descriptionsPath, 'utf-8'); + this.descriptionsData = JSON.parse(content); + return this.descriptionsData || {}; + } + + private loadRelatedTags(): string[][] { + if (this.relatedTagsData) { + return this.relatedTagsData; + } + + if (!fs.existsSync(this.relatedTagsPath)) { + return []; + } + + const content = fs.readFileSync(this.relatedTagsPath, 'utf-8'); + this.relatedTagsData = JSON.parse(content) || []; + return this.relatedTagsData || []; + } + + private capitalizeFirstLetter(str: string): string { + return str.charAt(0).toUpperCase() + str.slice(1); + } + + getAll(type: TagsType): ApiTag[] { + const tags = this.loadTags(); + + const languageTags: ApiTag[] = tags.languages.map((lang) => ({ + name: this.capitalizeFirstLetter(lang), + value: lang, + tag_type: 'languages', + })); + + const otherTags: ApiTag[] = tags.others.map((other) => ({ + name: this.capitalizeFirstLetter(other), + value: other, + tag_type: 'other', + })); + + switch (type) { + case 'languages': + return languageTags; + case 'other': + return otherTags; + case 'all': + return [...languageTags, ...otherTags]; + default: + console.error(`Unknown tag type: ${type}`); + return []; + } + } + + getLanguages(): string[] { + return this.loadTags().languages; + } + + getOthers(): string[] { + return this.loadTags().others; + } + + getById(type: TagsType, tagId: string): ApiTag | null { + const tags = this.getAll(type); + return ( + tags.find((t) => t.value.toLowerCase() === tagId.toLowerCase()) || + null + ); + } + + getDescription(tagId: string): LanguageData { + const defaultData: LanguageData = { + name: this.capitalizeFirstLetter(tagId), + website: '', + description: '', + }; + + const descriptions = this.loadDescriptions(); + const data = descriptions[tagId]; + + if (!data || !isLanguageData(data)) { + return defaultData; + } + + return data; + } + + getRelated(tag: string): string[] { + const relatedTags = this.loadRelatedTags(); + + const relatedGroup = relatedTags.find((tags) => + tags.includes(tag.toLowerCase()), + ); + + if (!relatedGroup) { + return []; + } + + return relatedGroup.filter((t) => t !== tag.toLowerCase()); + } + + clearCache(): void { + this.tagsData = null; + this.descriptionsData = null; + this.relatedTagsData = null; + } +} diff --git a/lib/repositories/ToolsFilter.ts b/lib/repositories/ToolsFilter.ts new file mode 100644 index 0000000..8fe789e --- /dev/null +++ b/lib/repositories/ToolsFilter.ts @@ -0,0 +1,217 @@ +/** + * ToolsFilter + * + * Filter class for filtering tools based on various criteria. + * Works with tools data from ToolsRepository. + */ + +import type { Tool } from '@components/tools/types'; +import type { ApiTool, ToolsApiData } from 'utils/types'; +import type { ParsedUrlQuery } from 'querystring'; + +export class ToolsFilter { + private tools: ToolsApiData; + + constructor(tools: ToolsApiData) { + this.tools = tools; + } + + static from(tools: ToolsApiData): ToolsFilter { + return new ToolsFilter(tools); + } + + private toToolArray(entries: [string, ApiTool][]): Tool[] { + return entries.map(([id, tool]) => ({ + ...tool, + id, + votes: tool.votes || 0, + })) as Tool[]; + } + + private isSingleLanguageTool(tool: ApiTool): boolean { + return tool.languages.length <= 2; + } + + private containsArray(arr: string[], values: string[]): boolean { + return values.every((value) => arr.includes(value)); + } + + byLanguage(language: string): Tool[] { + const entries = Object.entries(this.tools).filter(([, tool]) => + tool.languages.includes(language.toLowerCase()), + ); + return this.toToolArray(entries); + } + + byLanguages(languages: string[]): Tool[] { + const entries = Object.entries(this.tools).filter(([, tool]) => { + const isMultiLanguage = !this.isSingleLanguageTool(tool); + return ( + isMultiLanguage && this.containsArray(tool.languages, languages) + ); + }); + return this.toToolArray(entries); + } + + byCategory(category: string): Tool[] { + const entries = Object.entries(this.tools).filter(([, tool]) => + tool.categories.includes(category.toLowerCase()), + ); + return this.toToolArray(entries); + } + + byType(type: string): Tool[] { + const entries = Object.entries(this.tools).filter(([, tool]) => + tool.types.includes(type.toLowerCase()), + ); + return this.toToolArray(entries); + } + + byLicense(license: string): Tool[] { + const entries = Object.entries(this.tools).filter(([, tool]) => + tool.licenses.includes(license), + ); + return this.toToolArray(entries); + } + + byTag(tag: string): Tool[] { + const entries = Object.entries(this.tools).filter( + ([, tool]) => + tool.languages.includes(tag) || tool.other.includes(tag), + ); + return this.toToolArray(entries); + } + + byTags(tags: string[]): Tool[] { + const entries = Object.entries(this.tools).filter(([, tool]) => { + const matchesLanguage = tags.some((t) => + tool.languages.includes(t), + ); + const matchesOther = tags.some((t) => tool.other.includes(t)); + return matchesLanguage || matchesOther; + }); + return this.toToolArray(entries); + } + + byQuery(query: ParsedUrlQuery): Tool[] { + const { languages, others, categories, types, licenses, pricing } = + query; + const result: Tool[] = []; + + for (const [key, tool] of Object.entries(this.tools)) { + if (languages && !this.matchesLanguageFilter(tool, languages)) { + continue; + } + + if (others && !this.matchesOthersFilter(tool, others)) { + continue; + } + + if ( + categories && + !this.matchesArrayFilter(tool.categories, categories) + ) { + continue; + } + + if (types && !this.matchesArrayFilter(tool.types, types)) { + continue; + } + + if (licenses && !this.matchesArrayFilter(tool.licenses, licenses)) { + continue; + } + + if (pricing && !this.matchesPricingFilter(tool, pricing)) { + continue; + } + + result.push({ + ...tool, + id: key, + votes: tool.votes || 0, + } as Tool); + } + + return result; + } + + private matchesLanguageFilter( + tool: ApiTool, + filter: string | string[], + ): boolean { + if (Array.isArray(filter)) { + const isMultiLanguage = !this.isSingleLanguageTool(tool); + return ( + isMultiLanguage && this.containsArray(tool.languages, filter) + ); + } + return tool.languages.includes(filter); + } + + private matchesOthersFilter( + tool: ApiTool, + filter: string | string[], + ): boolean { + if (Array.isArray(filter)) { + const isMultiLanguage = !this.isSingleLanguageTool(tool); + return isMultiLanguage && this.containsArray(tool.other, filter); + } + return tool.other.includes(filter); + } + + private matchesArrayFilter( + toolValues: string[], + filter: string | string[], + ): boolean { + if (Array.isArray(filter)) { + return this.containsArray(toolValues, filter); + } + return toolValues.includes(filter); + } + + private matchesPricingFilter( + tool: ApiTool, + pricing: string | string[], + ): boolean { + const pricingArray = Array.isArray(pricing) ? pricing : [pricing]; + + for (const filter of pricingArray) { + if (filter === 'plans' && !tool.plans) { + return false; + } + if (filter === 'oss' && !tool.plans?.oss) { + return false; + } + if (filter === 'free' && !tool.plans?.free) { + return false; + } + } + + return true; + } + + all(): Tool[] { + return this.toToolArray(Object.entries(this.tools)); + } + + sortByVotes(tools: Tool[]): Tool[] { + return [...tools].sort((a, b) => (b.votes || 0) - (a.votes || 0)); + } + + sortByName(tools: Tool[]): Tool[] { + return [...tools].sort((a, b) => a.name.localeCompare(b.name)); + } + + paginate( + tools: Tool[], + offset: number, + limit: number, + ): { data: Tool[]; nextCursor?: number; total: number } { + const total = tools.length; + const data = tools.slice(offset, offset + limit); + const nextCursor = offset + limit < total ? offset + limit : undefined; + + return { data, nextCursor, total }; + } +} diff --git a/lib/repositories/ToolsRepository.ts b/lib/repositories/ToolsRepository.ts new file mode 100644 index 0000000..b6d1a8c --- /dev/null +++ b/lib/repositories/ToolsRepository.ts @@ -0,0 +1,191 @@ +/** + * ToolsRepository + * + * Repository class for accessing tools data from static JSON files. + * Data is pre-built at build time by scripts/build-data.ts. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { ToolsApiData, ApiTool, VotesApiData } from 'utils/types'; +import type { Tool } from '@components/tools/types'; +import { calculateUpvotePercentage } from 'utils/votes'; + +interface BuildMeta { + buildTime: string; + staticAnalysisCount: number; + dynamicAnalysisCount: number; + totalCount: number; +} + +interface StaticToolsData { + tools: ToolsApiData; + meta: BuildMeta; +} + +export class ToolsRepository { + private static instance: ToolsRepository | null = null; + private toolsData: StaticToolsData | null = null; + private readonly dataPath: string; + + private constructor() { + this.dataPath = path.join(process.cwd(), 'data', 'tools.json'); + } + + static getInstance(): ToolsRepository { + if (!ToolsRepository.instance) { + ToolsRepository.instance = new ToolsRepository(); + } + return ToolsRepository.instance; + } + + private loadData(): StaticToolsData { + if (this.toolsData) { + return this.toolsData; + } + + if (!fs.existsSync(this.dataPath)) { + console.warn( + 'Static tools data not found. Run `npm run build-data` first.', + ); + return { tools: {}, meta: {} as BuildMeta }; + } + + const content = fs.readFileSync(this.dataPath, 'utf-8'); + this.toolsData = JSON.parse(content) as StaticToolsData; + return this.toolsData; + } + + getAll(): ToolsApiData { + return this.loadData().tools; + } + + getMeta(): BuildMeta | null { + return this.loadData().meta || null; + } + + getById(toolId: string): Tool | null { + const tools = this.getAll(); + const tool = tools[toolId]; + + if (!tool) { + return null; + } + + return { + ...tool, + id: toolId, + votes: tool.votes || 0, + } as Tool; + } + + getAllIds(): string[] { + return Object.keys(this.getAll()); + } + + toArray(): Tool[] { + const tools = this.getAll(); + return Object.entries(tools).map(([id, tool]) => ({ + ...tool, + id, + votes: tool.votes || 0, + })) as Tool[]; + } + + findWhere(predicate: (tool: ApiTool, id: string) => boolean): Tool[] { + const tools = this.getAll(); + return Object.entries(tools) + .filter(([id, tool]) => predicate(tool, id)) + .map(([id, tool]) => ({ + ...tool, + id, + votes: tool.votes || 0, + })) as Tool[]; + } + + findByLanguage(language: string): Tool[] { + return this.findWhere((tool) => + tool.languages.includes(language.toLowerCase()), + ); + } + + findByCategory(category: string): Tool[] { + return this.findWhere((tool) => + tool.categories.includes(category.toLowerCase()), + ); + } + + findByType(type: string): Tool[] { + return this.findWhere((tool) => + tool.types.includes(type.toLowerCase()), + ); + } + + findByTag(tag: string): Tool[] { + return this.findWhere( + (tool) => tool.languages.includes(tag) || tool.other.includes(tag), + ); + } + + count(): number { + return Object.keys(this.getAll()).length; + } + + getIcon(toolId: string): string | null { + const iconPath = path.join( + process.cwd(), + 'public', + 'assets', + 'images', + 'tools', + `${toolId}.png`, + ); + + if (fs.existsSync(iconPath)) { + return `/assets/images/tools/${toolId}.png`; + } + return null; + } + + withVotes(votes: VotesApiData | null): ToolsApiData { + const tools = this.getAll(); + + if (!votes) { + return tools; + } + + const result: ToolsApiData = {}; + + for (const [toolId, tool] of Object.entries(tools)) { + const key = `toolsyaml${toolId}`; + const v = votes[key]; + + const sum = v?.sum || 0; + const upVotes = v?.upVotes || 0; + const downVotes = v?.downVotes || 0; + + result[toolId] = { + ...tool, + votes: sum, + upVotes, + downVotes, + upvotePercentage: calculateUpvotePercentage(upVotes, downVotes), + }; + } + + return result; + } + + withVotesAsArray(votes: VotesApiData | null): Tool[] { + const tools = this.withVotes(votes); + return Object.entries(tools).map(([id, tool]) => ({ + ...tool, + id, + votes: tool.votes || 0, + })) as Tool[]; + } + + clearCache(): void { + this.toolsData = null; + } +} diff --git a/lib/repositories/VotesRepository.ts b/lib/repositories/VotesRepository.ts new file mode 100644 index 0000000..9ae3df3 --- /dev/null +++ b/lib/repositories/VotesRepository.ts @@ -0,0 +1,128 @@ +/** + * VotesRepository + * + * Repository class for accessing votes data from Firebase. + * Provides a simplified interface for fetching votes at build time. + */ + +import type { VotesApiData } from 'utils/types'; + +export class VotesRepository { + private static instance: VotesRepository | null = null; + private initPromise: Promise | null = null; + private votesCache: VotesApiData | null = null; + + // Private constructor for singleton pattern + // eslint-disable-next-line @typescript-eslint/no-empty-function + private constructor() {} + + static getInstance(): VotesRepository { + if (!VotesRepository.instance) { + VotesRepository.instance = new VotesRepository(); + } + return VotesRepository.instance; + } + + private async initFirebase(): Promise { + if (this.initPromise) { + return this.initPromise; + } + + this.initPromise = (async () => { + const { apps, credential } = await import('firebase-admin'); + const { initializeApp } = await import('firebase-admin/app'); + + if (!apps.length) { + initializeApp({ + credential: credential.applicationDefault(), + databaseURL: 'https://analysis-tools-dev.firebaseio.com', + }); + } + })(); + + return this.initPromise; + } + + isConfigured(): boolean { + return !!process.env.GOOGLE_APPLICATION_CREDENTIALS; + } + + async fetchAll(): Promise { + if (!this.isConfigured()) { + console.warn( + 'Firebase credentials not configured. Skipping votes fetch.', + ); + return null; + } + + if (this.votesCache) { + return this.votesCache; + } + + try { + await this.initFirebase(); + + const { getFirestore } = await import('firebase-admin/firestore'); + const db = getFirestore(); + + const votesCol = db.collection('tags'); + const voteSnapshot = await votesCol.get(); + + const votes: VotesApiData = {}; + voteSnapshot.docs.forEach((doc) => { + const data = doc.data(); + votes[doc.id] = { + sum: data.sum || 0, + upVotes: data.upVotes || 0, + downVotes: data.downVotes || 0, + }; + }); + + this.votesCache = votes; + return votes; + } catch (error) { + console.error('Error fetching votes from Firebase:', error); + return null; + } + } + + async fetchForTool(toolId: string): Promise<{ + votes: number; + upVotes: number; + downVotes: number; + }> { + const defaultVotes = { votes: 0, upVotes: 0, downVotes: 0 }; + + if (!this.isConfigured()) { + return defaultVotes; + } + + try { + await this.initFirebase(); + + const { getFirestore } = await import('firebase-admin/firestore'); + const db = getFirestore(); + + const key = `toolsyaml${toolId}`; + const doc = await db.collection('tags').doc(key).get(); + + if (!doc.exists) { + return defaultVotes; + } + + const data = doc.data(); + return { + votes: data?.sum || 0, + upVotes: data?.upVotes || 0, + downVotes: data?.downVotes || 0, + }; + } catch (error) { + console.error(`Error fetching votes for tool ${toolId}:`, error); + return defaultVotes; + } + } + + clearCache(): void { + this.votesCache = null; + } +} diff --git a/lib/repositories/index.ts b/lib/repositories/index.ts new file mode 100644 index 0000000..3ec60ed --- /dev/null +++ b/lib/repositories/index.ts @@ -0,0 +1,5 @@ +export { ToolsRepository } from './ToolsRepository'; +export { TagsRepository } from './TagsRepository'; +export { StatsRepository } from './StatsRepository'; +export { VotesRepository } from './VotesRepository'; +export { ToolsFilter } from './ToolsFilter'; diff --git a/package.json b/package.json index 624a156..bb68db1 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,8 @@ "private": true, "scripts": { "dev": "next dev", + "build-data": "ts-node -r tsconfig-paths/register --compiler-options '{\"module\":\"CommonJS\"}' ./scripts/build-data.ts", + "prebuild": "npm run build-data", "build": "next build", "search-index": "ts-node -r tsconfig-paths/register --compiler-options '{\"module\":\"CommonJS\"}' ./algolia-index.ts", "start": "next start", diff --git a/pages/index.tsx b/pages/index.tsx index 7f75cc7..384a8e5 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -14,17 +14,21 @@ import homepageData from '@appdata/homepage.json'; import { ArticlePreview, Faq, SponsorData } from 'utils/types'; import { Tool, ToolsByLanguage } from '@components/tools'; import { getArticlesPreviews } from 'utils-api/blog'; -import { getPopularLanguageStats } from 'utils-api/popularLanguageStats'; -import { getMostViewedTools } from 'utils-api/mostViewedTools'; import { getSponsors } from 'utils-api/sponsors'; import { getFaq } from 'utils-api/faq'; +import { StatsRepository, VotesRepository } from '@lib/repositories'; export const getStaticProps: GetStaticProps = async () => { const sponsors = getSponsors(); const faq = getFaq(); const previews = await getArticlesPreviews(); - const popularLanguages = await getPopularLanguageStats(); - const mostViewed = await getMostViewedTools(); + + const votesRepo = VotesRepository.getInstance(); + const statsRepo = StatsRepository.getInstance(); + + const votes = await votesRepo.fetchAll(); + const popularLanguages = statsRepo.getPopularLanguageStats(votes); + const mostViewed = statsRepo.getMostViewedTools(votes); return { props: { diff --git a/pages/tag/[slug].tsx b/pages/tag/[slug].tsx index bcf67d0..367edc5 100644 --- a/pages/tag/[slug].tsx +++ b/pages/tag/[slug].tsx @@ -12,34 +12,31 @@ import { SponsorData, } from 'utils/types'; import { getArticlesPreviews } from 'utils-api/blog'; -import { getLanguageData, getSimilarTags, getTags } from 'utils-api/tags'; -import { filterByTags } from 'utils-api/filters'; import { getSponsors } from 'utils-api/sponsors'; -import { getToolsWithVotes } from 'utils-api/toolsWithVotes'; import { RelatedTagsList } from '@components/tools/listPage/RelatedTagsList'; import { LanguageFilterOption } from '@components/tools/listPage/ToolsSidebar/FilterCard/LanguageFilterCard'; import { Tool } from '@components/tools'; import { TagsSidebar } from '@components/tags'; import { getRandomAffiliate } from 'utils-api/affiliates'; +import { + TagsRepository, + ToolsRepository, + ToolsFilter, + VotesRepository, +} from '@lib/repositories'; -// This function gets called at build time export const getStaticPaths: GetStaticPaths = async () => { - // Call an external API endpoint to get tools - const data = await getTags('all'); + const tagsRepo = TagsRepository.getInstance(); + const tags = tagsRepo.getAll('all'); - if (!data) { + if (tags.length === 0) { return { paths: [], fallback: false }; } - // Get the paths we want to pre-render based on the tags API response - const paths = data.map((tag) => { - return { - params: { slug: tag.value }, - }; - }); + const paths = tags.map((tag) => ({ + params: { slug: tag.value }, + })); - // We'll pre-render only these paths at build time. - // { fallback: false } means other routes should 404. return { paths, fallback: false }; }; @@ -51,22 +48,28 @@ export const getStaticProps: GetStaticProps = async ({ params }) => { }; } - const tagData = await getLanguageData(slug); + const tagsRepo = TagsRepository.getInstance(); + const toolsRepo = ToolsRepository.getInstance(); + const votesRepo = VotesRepository.getInstance(); + + const tagData = tagsRepo.getDescription(slug); - // Capitalize the first letter of the tag let tagName = slug.charAt(0).toUpperCase() + slug.slice(1); if ('name' in tagData && tagData.name) { - // We can use a more descriptive name if it exists tagName = tagData.name; } - const tools = await getToolsWithVotes(); + + const votes = await votesRepo.fetchAll(); + const toolsWithVotes = toolsRepo.withVotes(votes); + const filter = ToolsFilter.from(toolsWithVotes); + const previews = await getArticlesPreviews(); const sponsors = getSponsors(); - const languages = await getTags('languages'); + const languages = tagsRepo.getAll('languages'); const affiliate = getRandomAffiliate([slug]); - const relatedTags = getSimilarTags(slug); - const filteredTools = filterByTags(tools, slug); + const relatedTags = tagsRepo.getRelated(slug); + const filteredTools = filter.byTag(slug); // best tools by percent upvoted (single language only) const bestTools = filteredTools diff --git a/pages/tool/[slug].tsx b/pages/tool/[slug].tsx index e3bd1f3..223a333 100644 --- a/pages/tool/[slug].tsx +++ b/pages/tool/[slug].tsx @@ -2,7 +2,6 @@ import { FC } from 'react'; import { GetStaticPaths, GetStaticProps } from 'next'; import { MainHead, Footer, Navbar, SponsorBanner } from '@components/core'; import { Main, Panel, Wrapper } from '@components/layout'; -import { getTool, getToolIcon } from 'utils-api/tools'; import { AlternativeToolsList, Tool, @@ -11,32 +10,28 @@ import { } from '@components/tools'; import { SearchProvider } from 'context/SearchProvider'; import { getScreenshots } from 'utils-api/screenshot'; -import { getAllTools } from 'utils-api/tools'; import { ArticlePreview, SponsorData, StarHistory } from 'utils/types'; import { containsArray } from 'utils/arrays'; -import { getVotes } from 'utils-api/votes'; import { getArticlesPreviews } from 'utils-api/blog'; import { getSponsors } from 'utils-api/sponsors'; import { ToolGallery } from '@components/tools/toolPage/ToolGallery'; import { Comments } from '@components/core/Comments'; import { calculateUpvotePercentage } from 'utils/votes'; +import { ToolsRepository, VotesRepository } from '@lib/repositories'; // This function gets called at build time export const getStaticPaths: GetStaticPaths = async () => { - // Call an external API endpoint to get tools - const data = await getAllTools(); + const toolsRepo = ToolsRepository.getInstance(); + const toolIds = toolsRepo.getAllIds(); - if (!data) { + if (toolIds.length === 0) { return { paths: [], fallback: false }; } - // Get the paths we want to pre-render based on the tools API response - const paths = Object.keys(data).map((id) => ({ + const paths = toolIds.map((id) => ({ params: { slug: id }, })); - // We'll pre-render only these paths at build time. - // { fallback: false } means other routes should 404. return { paths, fallback: false }; }; @@ -50,7 +45,10 @@ export const getStaticProps: GetStaticProps = async ({ params }) => { }; } - const apiTool = await getTool(slug); + const toolsRepo = ToolsRepository.getInstance(); + const votesRepo = VotesRepository.getInstance(); + + const apiTool = toolsRepo.getById(slug); if (!apiTool) { console.error(`Tool ${slug} not found. Cannot build slug page.`); return { @@ -59,12 +57,12 @@ export const getStaticProps: GetStaticProps = async ({ params }) => { } const sponsors = getSponsors(); - const votes = await getVotes(); + const votes = await votesRepo.fetchAll(); const previews = await getArticlesPreviews(); - const icon = getToolIcon(slug); + const icon = toolsRepo.getIcon(slug); - // calculate the upvote percentage based on the votes - const voteKey = `toolsyaml${slug.toString()}`; + // Calculate the upvote percentage based on the votes + const voteKey = `toolsyaml${slug}`; const voteData = votes ? votes[voteKey] : null; const upvotePercentage = calculateUpvotePercentage( voteData?.upVotes, @@ -77,70 +75,42 @@ export const getStaticProps: GetStaticProps = async ({ params }) => { id: slug, icon: icon, }; - const alternativeTools = await getAllTools(); + + // Get all tools with votes for alternatives + const allToolsWithVotes = toolsRepo.withVotesAsArray(votes); let alternatives: Tool[] = []; - let allAlternatives: Tool[] = []; - if (alternativeTools) { - allAlternatives = Object.entries(alternativeTools).reduce( - (acc, [id, tool]) => { - // if key is not equal to slug - if (id !== slug) { - // push value to acc - // add id and votes to value - const voteKey = `toolsyaml${id.toString()}`; - - // check if we have votes for this tool - // otherwise set to 0 - const voteData = votes - ? votes[voteKey] - ? votes[voteKey].sum - : 0 - : 0; - - acc.push({ id, ...tool, votes: voteData }); - } - return acc; - }, - [] as Tool[], + const allAlternatives = allToolsWithVotes.filter((t) => t.id !== slug); + + // Show only tools with the same type, languages, and categories + alternatives = allAlternatives.filter((alt) => { + return ( + containsArray(alt.types, tool.types || []) && + containsArray(alt.languages, tool.languages || []) && + containsArray(alt.categories, tool.categories || []) ); + }); - // if in currentTool view, show only tools with the same type - if (tool) { - alternatives = allAlternatives.filter((alt) => { - return ( - containsArray(alt.types, tool.types || []) && - containsArray(alt.languages, tool.languages || []) && - containsArray(alt.categories, tool.categories || []) - ); + // If the list is empty, show tools with most matched languages + if (alternatives.length === 0) { + alternatives = allAlternatives + .sort((a, b) => { + const aMatches = + a.languages?.filter((lang) => + tool.languages?.includes(lang), + ).length || 0; + const bMatches = + b.languages?.filter((lang) => + tool.languages?.includes(lang), + ).length || 0; + return bMatches - aMatches; + }) + .filter((alt) => { + const matches = + alt.languages?.filter((lang) => + tool.languages?.includes(lang), + ).length || 0; + return matches >= 5; }); - - // if the list is empty, show the tools with the same type and the most - // matched languages - if (alternatives.length === 0) { - alternatives = allAlternatives; - - // sort the list by the number of matched languages - alternatives.sort((a, b) => { - return ( - b.languages?.filter((lang) => - tool.languages?.includes(lang), - ).length - - a.languages?.filter((lang) => - tool.languages?.includes(lang), - ).length - ); - }); - - // take the tools with at least 5 matched languages - alternatives = alternatives.filter((alt) => { - return ( - alt.languages?.filter((lang) => - tool.languages?.includes(lang), - ).length >= 5 - ); - }); - } - } } return { diff --git a/scripts/build-data.ts b/scripts/build-data.ts new file mode 100644 index 0000000..7c1422f --- /dev/null +++ b/scripts/build-data.ts @@ -0,0 +1,293 @@ +/** + * Build-time data fetching script + * + * This script fetches tools data from the analysis-tools-dev GitHub repositories + * and consolidates them into a single static JSON file for use at runtime. + * + * Run with: npm run build-data + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as https from 'https'; + +// Types +interface ToolData { + name: string; + categories: string[]; + languages: string[]; + other: string[]; + licenses: string[]; + types: string[]; + homepage: string; + source: string | null; + pricing: string | null; + plans: { free?: boolean; oss?: boolean } | null; + description: string | null; + discussion: string | null; + deprecated: boolean | null; + resources: { title: string; url: string }[] | null; + reviews: unknown | null; + demos: unknown | null; + wrapper: string | null; +} + +interface ToolsData { + [key: string]: ToolData; +} + +interface StatsData { + [key: string]: number; +} + +interface BuildOutput { + tools: ToolsData; + meta: { + buildTime: string; + staticAnalysisCount: number; + dynamicAnalysisCount: number; + totalCount: number; + }; +} + +// GitHub raw content URLs +const STATIC_ANALYSIS_URL = + 'https://raw.githubusercontent.com/analysis-tools-dev/static-analysis/master/data/api/tools.json'; +const DYNAMIC_ANALYSIS_URL = + 'https://raw.githubusercontent.com/analysis-tools-dev/dynamic-analysis/master/data/api/tools.json'; +const TOOL_STATS_URL = + 'https://raw.githubusercontent.com/analysis-tools-dev/static-analysis/master/data/api/stats/tools.json'; +const TAG_STATS_URL = + 'https://raw.githubusercontent.com/analysis-tools-dev/static-analysis/master/data/api/stats/tags.json'; + +// Output paths +const OUTPUT_DIR = path.join(process.cwd(), 'data'); +const OUTPUT_FILE = path.join(OUTPUT_DIR, 'tools.json'); +const TOOL_STATS_FILE = path.join(OUTPUT_DIR, 'tool-stats.json'); +const TAG_STATS_FILE = path.join(OUTPUT_DIR, 'tag-stats.json'); + +/** + * Fetch JSON data from a URL + */ +function fetchJSON(url: string): Promise { + return new Promise((resolve, reject) => { + console.log(`Fetching: ${url}`); + + https + .get(url, (res) => { + if (res.statusCode !== 200) { + reject( + new Error( + `Failed to fetch ${url}: HTTP ${res.statusCode}`, + ), + ); + return; + } + + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + res.on('end', () => { + try { + const parsed = JSON.parse(data) as T; + resolve(parsed); + } catch (e) { + reject(new Error(`Failed to parse JSON from ${url}`)); + } + }); + }) + .on('error', (e) => { + reject(new Error(`Failed to fetch ${url}: ${e.message}`)); + }); + }); +} + +/** + * Merge tools from multiple sources, handling duplicates + */ +function mergeTools( + staticTools: ToolsData, + dynamicTools: ToolsData, +): ToolsData { + const merged: ToolsData = { ...staticTools }; + + for (const [id, tool] of Object.entries(dynamicTools)) { + if (merged[id]) { + // Tool exists in both - merge categories and types + console.log(`Merging duplicate tool: ${id}`); + merged[id] = { + ...merged[id], + categories: [ + ...new Set([...merged[id].categories, ...tool.categories]), + ], + types: [...new Set([...merged[id].types, ...tool.types])], + }; + } else { + merged[id] = tool; + } + } + + return merged; +} + +/** + * Validate tool data + */ +function validateTools(tools: ToolsData): void { + const errors: string[] = []; + + for (const [id, tool] of Object.entries(tools)) { + if (!tool.name) { + errors.push(`Tool "${id}" is missing a name`); + } + if (!Array.isArray(tool.categories)) { + errors.push(`Tool "${id}" has invalid categories`); + } + if (!Array.isArray(tool.languages)) { + errors.push(`Tool "${id}" has invalid languages`); + } + if (!Array.isArray(tool.types)) { + errors.push(`Tool "${id}" has invalid types`); + } + } + + if (errors.length > 0) { + console.warn('Validation warnings:'); + errors.forEach((e) => console.warn(` - ${e}`)); + } +} + +/** + * Extract unique tags (languages and others) from tools + */ +function extractTags(tools: ToolsData): { + languages: string[]; + others: string[]; +} { + const languages = new Set(); + const others = new Set(); + + for (const tool of Object.values(tools)) { + tool.languages?.forEach((lang) => languages.add(lang)); + tool.other?.forEach((other) => others.add(other)); + } + + return { + languages: Array.from(languages).sort(), + others: Array.from(others).sort(), + }; +} + +/** + * Fetch stats data (tool views, tag views) + */ +async function fetchStats(): Promise<{ + toolStats: StatsData; + tagStats: StatsData; +}> { + try { + const [toolStats, tagStats] = await Promise.all([ + fetchJSON(TOOL_STATS_URL), + fetchJSON(TAG_STATS_URL), + ]); + + // Normalize tag stats keys (remove /tag/ prefix) + const normalizedTagStats: StatsData = {}; + for (const [key, value] of Object.entries(tagStats)) { + const normalizedKey = key.replace('/tag/', ''); + normalizedTagStats[normalizedKey] = value; + } + + return { toolStats, tagStats: normalizedTagStats }; + } catch (error) { + console.warn('Warning: Could not fetch stats data:', error); + return { toolStats: {}, tagStats: {} }; + } +} + +/** + * Main build function + */ +async function main(): Promise { + console.log('Building tools data...\n'); + + try { + // Fetch data from both repositories and stats + const [staticTools, dynamicTools, stats] = await Promise.all([ + fetchJSON(STATIC_ANALYSIS_URL), + fetchJSON(DYNAMIC_ANALYSIS_URL), + fetchStats(), + ]); + + const staticCount = Object.keys(staticTools).length; + const dynamicCount = Object.keys(dynamicTools).length; + + console.log(`\nFetched ${staticCount} static analysis tools`); + console.log(`Fetched ${dynamicCount} dynamic analysis tools`); + + // Merge tools + const mergedTools = mergeTools(staticTools, dynamicTools); + const totalCount = Object.keys(mergedTools).length; + console.log(`Total unique tools: ${totalCount}`); + + // Validate + validateTools(mergedTools); + + // Extract tags for reference + const tags = extractTags(mergedTools); + console.log(`\nFound ${tags.languages.length} unique languages`); + console.log(`Found ${tags.others.length} unique other tags`); + + // Stats info + const toolStatsCount = Object.keys(stats.toolStats).length; + const tagStatsCount = Object.keys(stats.tagStats).length; + console.log(`\nFetched stats for ${toolStatsCount} tools`); + console.log(`Fetched stats for ${tagStatsCount} tags`); + + // Prepare output + const output: BuildOutput = { + tools: mergedTools, + meta: { + buildTime: new Date().toISOString(), + staticAnalysisCount: staticCount, + dynamicAnalysisCount: dynamicCount, + totalCount, + }, + }; + + // Ensure output directory exists + if (!fs.existsSync(OUTPUT_DIR)) { + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + } + + // Write output file + fs.writeFileSync(OUTPUT_FILE, JSON.stringify(output, null, 2)); + console.log(`\nWritten to ${OUTPUT_FILE}`); + + // Also write tags for reference (useful for filters) + const tagsFile = path.join(OUTPUT_DIR, 'tags.json'); + fs.writeFileSync(tagsFile, JSON.stringify(tags, null, 2)); + console.log(`Written tags to ${tagsFile}`); + + // Write stats files + fs.writeFileSync( + TOOL_STATS_FILE, + JSON.stringify(stats.toolStats, null, 2), + ); + console.log(`Written tool stats to ${TOOL_STATS_FILE}`); + + fs.writeFileSync( + TAG_STATS_FILE, + JSON.stringify(stats.tagStats, null, 2), + ); + console.log(`Written tag stats to ${TAG_STATS_FILE}`); + + console.log('\nBuild complete!'); + } catch (error) { + console.error('\nBuild failed:', error); + process.exit(1); + } +} + +main(); diff --git a/tsconfig.json b/tsconfig.json index 1257693..d417562 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,7 +18,8 @@ "baseUrl": ".", "paths": { "@components/*": ["components/*"], - "@appdata/*": ["data/*"] + "@appdata/*": ["data/*"], + "@lib/*": ["lib/*"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], diff --git a/utils/types.ts b/utils/types.ts index eea4b01..71af431 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -101,7 +101,7 @@ export interface LanguageTag { } export interface StatsApiData { - [key: string]: string; + [key: string]: number; } export interface VotesApiData {