diff --git a/.env b/.env index 8670a51..8877cae 100644 --- a/.env +++ b/.env @@ -1,4 +1,7 @@ # Crowdin CROWDIN_PROJECT_ID="op://Environment Variables - Development/Scripts/Crowdin Project ID" CROWDIN_API_URL="op://Environment Variables - Development/Scripts/Crowdin API Url" -CROWDIN_TOKEN="op://Environment Variables - Development/Scripts/Crowdin Token" \ No newline at end of file +CROWDIN_TOKEN="op://Environment Variables - Development/Scripts/Crowdin Token" + +# Github +GITHUB_TOKEN="op://Environment Variables - Development/Scripts/GitHub Token" \ No newline at end of file diff --git a/.gitignore b/.gitignore index aab8b24..a08a1ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ node_modules -prod \ No newline at end of file +prod +data +!data/.gitkeep \ No newline at end of file diff --git a/package.json b/package.json index fbd5901..2d682fe 100644 --- a/package.json +++ b/package.json @@ -21,5 +21,8 @@ "eslint": "9.34.0", "tsx": "4.20.5", "typescript": "5.9.2" + }, + "dependencies": { + "@octokit/rest": "22.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44cd442..f142bda 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,10 @@ settings: importers: .: + dependencies: + '@octokit/rest': + specifier: 22.0.0 + version: 22.0.0 devDependencies: '@nhcarrigan/eslint-config': specifier: 5.2.0 @@ -309,6 +313,58 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.3': + resolution: {integrity: sha512-oNXsh2ywth5aowwIa7RKtawnkdH6LgU1ztfP9AIUCQCvzysB+WeU8o2kyyosDPwBZutPpjZDKPQGIzzrfTWweQ==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.0': + resolution: {integrity: sha512-hoYicJZaqISMAI3JfaDr1qMNi48OctWuOih1m80bkYow/ayPw6Jj52tqWJ6GEoFTk1gBqfanSoI1iY99Z5+ekQ==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.1': + resolution: {integrity: sha512-j1nQNU1ZxNFx2ZtKmL4sMrs4egy5h65OMDmSbVyuCzjOcwsHq6EaYjOTGXPQxgfiN8dJ4CriYHk6zF050WEULg==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@25.1.0': + resolution: {integrity: sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA==} + + '@octokit/plugin-paginate-rest@13.1.1': + resolution: {integrity: sha512-q9iQGlZlxAVNRN2jDNskJW/Cafy7/XE52wjZ5TTvyhyOD904Cvx//DNyoO3J/MXJ0ve3rPoNWKEg5iZrisQSuw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@16.0.0': + resolution: {integrity: sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.0.0': + resolution: {integrity: sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.3': + resolution: {integrity: sha512-V6jhKokg35vk098iBqp2FBKunk3kMTXlmq+PtbV9Gl3TfskWlebSofU9uunVKhUN7xl+0+i5vt0TGTG8/p/7HA==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.0': + resolution: {integrity: sha512-z6tmTu9BTnw51jYGulxrlernpsQYXpui1RK21vmXn8yF5bp6iX16yfTtJYGK5Mh1qDkvDOmp2n8sRMcQmR8jiA==} + engines: {node: '>= 20'} + + '@octokit/types@14.1.0': + resolution: {integrity: sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g==} + '@pkgr/core@0.1.2': resolution: {integrity: sha512-fdDH1LSGfZdTH2sxdpVMw31BanV28K/Gry0cVFxaNP77neJSkd82mM8ErPNYs9e+0O7SdHBLTDzDgwUuy18RnQ==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -676,6 +732,9 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -998,6 +1057,9 @@ packages: resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} engines: {node: '>=12.0.0'} + fast-content-type-parse@3.0.0: + resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1847,6 +1909,9 @@ packages: undici-types@7.10.0: resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==} + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + update-browserslist-db@1.1.3: resolution: {integrity: sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==} hasBin: true @@ -2190,6 +2255,68 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.3': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.1 + '@octokit/request': 10.0.3 + '@octokit/request-error': 7.0.0 + '@octokit/types': 14.1.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.0': + dependencies: + '@octokit/types': 14.1.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.1': + dependencies: + '@octokit/request': 10.0.3 + '@octokit/types': 14.1.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@25.1.0': {} + + '@octokit/plugin-paginate-rest@13.1.1(@octokit/core@7.0.3)': + dependencies: + '@octokit/core': 7.0.3 + '@octokit/types': 14.1.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.3)': + dependencies: + '@octokit/core': 7.0.3 + + '@octokit/plugin-rest-endpoint-methods@16.0.0(@octokit/core@7.0.3)': + dependencies: + '@octokit/core': 7.0.3 + '@octokit/types': 14.1.0 + + '@octokit/request-error@7.0.0': + dependencies: + '@octokit/types': 14.1.0 + + '@octokit/request@10.0.3': + dependencies: + '@octokit/endpoint': 11.0.0 + '@octokit/request-error': 7.0.0 + '@octokit/types': 14.1.0 + fast-content-type-parse: 3.0.0 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.0': + dependencies: + '@octokit/core': 7.0.3 + '@octokit/plugin-paginate-rest': 13.1.1(@octokit/core@7.0.3) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.3) + '@octokit/plugin-rest-endpoint-methods': 16.0.0(@octokit/core@7.0.3) + + '@octokit/types@14.1.0': + dependencies: + '@octokit/openapi-types': 25.1.0 + '@pkgr/core@0.1.2': {} '@rollup/rollup-android-arm-eabi@4.49.0': @@ -2607,6 +2734,8 @@ snapshots: balanced-match@1.0.2: {} + before-after-hook@4.0.0: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -3102,6 +3231,8 @@ snapshots: expect-type@1.2.2: {} + fast-content-type-parse@3.0.0: {} + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -4014,6 +4145,8 @@ snapshots: undici-types@7.10.0: {} + universal-user-agent@7.0.3: {} + update-browserslist-db@1.1.3(browserslist@4.25.3): dependencies: browserslist: 4.25.3 diff --git a/src/crowdin/clearHiddenTranslations.ts b/src/crowdin/clearHiddenTranslations.ts new file mode 100644 index 0000000..325dff4 --- /dev/null +++ b/src/crowdin/clearHiddenTranslations.ts @@ -0,0 +1,75 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +import { readFile, appendFile } from "node:fs/promises"; +import { join } from "node:path"; +import { getLanguages } from "./utils/getLanguages.js"; +import type { String } from "./interfaces/string.js"; + +const projectId = process.env.CROWDIN_PROJECT_ID; +const apiUrl = process.env.CROWDIN_API_URL; +const token = process.env.CROWDIN_TOKEN; + +if ( + projectId === undefined + || projectId === "" + || apiUrl === undefined + || apiUrl === "" + || token === undefined + || token === "" +) { + throw new Error(`Project ID or API URL is missing! Did you run this script with 'op run'?`); +} + +const logPath + = join(import.meta.dirname, "..", "..", "data", "crowdin-strings-hidden.txt"); + +const languages = await getLanguages(projectId, apiUrl, token); +console.log(`Found ${languages.length.toString()} active languages.`); +const rawStrings = await readFile( + join(import.meta.dirname, "..", "..", "data", "crowdin-strings.json"), + "utf-8", +); +const strings: Array = JSON.parse(rawStrings); +console.log(`Found ${strings.length.toString()} strings.`); + +const log = await readFile( + logPath, + "utf-8", +); + +const processedIds = log.split("\n"); + +const unprocessedStrings = strings.filter((string) => { + return !processedIds.includes(string.id.toString()); +}); + +const hidden = unprocessedStrings.filter((string) => { + return string.isHidden; +}); + +console.log(`Processing ${hidden.length.toString()} hidden strings.`); + +for (const string of hidden) { + console.log(`Deleting translations for ${string.id.toString()}`); + await Promise.all(languages.map(async(language) => { + await fetch(`${apiUrl}/projects/${projectId}/translations`, { + body: JSON.stringify({ + languageId: language, + stringId: string.id, + }), + headers: { + "authorization": `Bearer ${token}`, + // eslint-disable-next-line @typescript-eslint/naming-convention -- header. + "content-type": "application/json", + }, + method: "DELETE", + }); + })); + await appendFile(logPath, `${string.id.toString()}\n`); +} + +console.log("Completed!"); diff --git a/src/crowdin/interfaces/string.ts b/src/crowdin/interfaces/string.ts new file mode 100644 index 0000000..8f38939 --- /dev/null +++ b/src/crowdin/interfaces/string.ts @@ -0,0 +1,36 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +export interface String { + data: Array<{ + data: { + id: number; + projectId: number; + branchId: number; + identifier: string; + text: string; + type: string; + context: string; + maxLength: number; + isHidden: boolean; + isDuplicate: boolean; + masterStringId: number; + hasPlurals: boolean; + isIcu: boolean; + labelIds: Array; + webUrl: string; + createdAt: string; + updatedAt: string; + fileId: number; + directoryId: number; + revision: number; + }; + }>; + pagination: { + offset: number; + limit: number; + }; +} diff --git a/src/crowdin/utils/getFiles.ts b/src/crowdin/utils/getFiles.ts index 96cf57a..6595202 100644 --- a/src/crowdin/utils/getFiles.ts +++ b/src/crowdin/utils/getFiles.ts @@ -18,10 +18,10 @@ export const getFiles projectId: string, apiUrl: string, token: string, - ): Promise> => { + ): Promise> => { const url = `${apiUrl}/projects/${projectId}/files?limit=500`; let offset = 0; - const ids: Array = []; + const files: Array = []; console.log(`Requesting files ${offset.toString()} to ${(offset + 500).toString()}`); let request = await fetch(url, { @@ -30,9 +30,7 @@ export const getFiles }, }); let response: File = await request.json(); - ids.push(...response.data.map((datum) => { - return datum.data.id; - })); + files.push(...response.data); while (response.data.length >= 500) { offset = offset + 500; console.log(`Requesting files ${offset.toString()} to ${(offset + 500).toString()}`); @@ -42,9 +40,7 @@ export const getFiles }, }); response = await request.json(); - ids.push(...response.data.map((datum) => { - return datum.data.id; - })); + files.push(...response.data); } - return ids; + return files; }; diff --git a/src/crowdin/utils/getStrings.ts b/src/crowdin/utils/getStrings.ts new file mode 100644 index 0000000..aee8466 --- /dev/null +++ b/src/crowdin/utils/getStrings.ts @@ -0,0 +1,50 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +import type { String } from "../interfaces/string.js"; + +/** + * Gets a list of all strings from a project. + * @param projectId - The project ID (a numeric string). + * @param apiUrl - The base URL for the API. + * @param token - The API key. + * @returns An array of string objects. + */ +export const getStrings + = async( + projectId: string, + apiUrl: string, + token: string, + ): Promise> => { + const url = `${apiUrl}/projects/${projectId}/strings?limit=500`; + let offset = 0; + const strings: Array = []; + + console.log(`Requesting strings ${offset.toString()} to ${(offset + 500).toString()}`); + let request = await fetch(url, { + headers: { + authorization: `Bearer ${token}`, + }, + }); + let response: String = await request.json(); + strings.push(...response.data.map((datum) => { + return datum.data; + })); + while (response.data.length >= 500) { + offset = offset + 500; + console.log(`Requesting strings ${offset.toString()} to ${(offset + 500).toString()}`); + request = await fetch(`${url}&offset=${offset.toString()}`, { + headers: { + authorization: `Bearer ${token}`, + }, + }); + response = await request.json(); + strings.push(...response.data.map((datum) => { + return datum.data; + })); + } + return strings; + }; diff --git a/src/crowdin/writeData.ts b/src/crowdin/writeData.ts new file mode 100644 index 0000000..678ddae --- /dev/null +++ b/src/crowdin/writeData.ts @@ -0,0 +1,41 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { getFiles } from "./utils/getFiles.js"; +import { getStrings } from "./utils/getStrings.js"; + +const projectId = process.env.CROWDIN_PROJECT_ID; +const apiUrl = process.env.CROWDIN_API_URL; +const token = process.env.CROWDIN_TOKEN; + +if ( + projectId === undefined + || projectId === "" + || apiUrl === undefined + || apiUrl === "" + || token === undefined + || token === "" +) { + throw new Error(`Project ID or API URL is missing! Did you run this script with 'op run'?`); +} + +const files = await getFiles(projectId, apiUrl, token); +const strings = await getStrings(projectId, apiUrl, token); + +await writeFile( + join(import.meta.dirname, "..", "..", "data", "crowdin-files.json"), + JSON.stringify(files, null, 2), + "utf-8", +); +await writeFile( + join(import.meta.dirname, "..", "..", "data", "crowdin-strings.json"), + JSON.stringify(strings, null, 2), + "utf-8", +); + +console.log("Loaded files and strings!"); diff --git a/src/github/auditNpmPackages.ts b/src/github/auditNpmPackages.ts new file mode 100644 index 0000000..ecae1bd --- /dev/null +++ b/src/github/auditNpmPackages.ts @@ -0,0 +1,151 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +import { writeFile, appendFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Octokit } from "@octokit/rest"; +import { serialiseJsonOrError } from "../utils/serialiseJsonOrError.js"; + +if (process.env.GITHUB_TOKEN === undefined) { + throw new Error("Missing Github Token - did you run this with `op`?"); +} + +const resultPath = join( + import.meta.dirname, + "..", + "..", + "data", + "npm-vulnerabilities.txt", +); + +const orgsToCheck = [ + "deepgram", + "deepgram-devs", + "deepgram-starters", +]; + +const vulnerablePackages: Array<{ name: string; version: string }> = [ + { name: "ansi-regex", version: "6.2.1" }, + { name: "ansi-styles", version: "6.2.2" }, + { name: "backslash", version: "0.2.1" }, + { name: "chalk-template", version: "1.1.1" }, + { name: "chalk", version: "5.6.1" }, + { name: "color-convert", version: "3.1.1" }, + { name: "color-name", version: "2.0.1" }, + { name: "color-string", version: "2.1.1" }, + { name: "color", version: "5.0.1" }, + { name: "debug", version: "4.4.2" }, + { name: "has-ansi", version: "6.0.1" }, + { name: "is-arrayish", version: "0.3.3" }, + { name: "simple-swizzle", version: "0.2.3" }, + { name: "slice-ansi", version: "7.1.1" }, + { name: "strip-ansi", version: "7.1.1" }, + { name: "supports-color", version: "10.2.1" }, + { name: "supports-hyperlinks", version: "4.1.1" }, + { name: "wrap-ansi", version: "9.0.1" }, +]; + +const gh = new Octokit({ + auth: process.env.GITHUB_TOKEN, +}); + +const repositories: Array<{ name: string; owner: string }> + = []; + +await writeFile(resultPath, "", "utf-8"); + +for (const org of orgsToCheck) { + let page = 1; + console.log(`Fetching ${org} repositories page ${page.toString()}`); + let repos = await gh.repos.listForOrg( + { + org: org, + // eslint-disable-next-line @typescript-eslint/naming-convention -- SDK signature. + per_page: 100, + }, + ); + repositories.push(...repos.data.map((repo) => { + return { name: repo.name, owner: org }; + })); + while (repos.data.length >= 100) { + page = page + 1; + console.log(`Fetching ${org} repositories page ${page.toString()}`); + repos = await gh.repos.listForOrg({ + org: org, + page: page, + // eslint-disable-next-line @typescript-eslint/naming-convention -- SDK signature. + per_page: 100, + }); + repositories.push(...repos.data.map((repo) => { + return { name: repo.name, owner: org }; + })); + } +} + +console.log(`Found ${repositories.length.toString()} repositories in ${orgsToCheck.length.toString()} orgs.`); + +for (const repo of repositories) { + const fileRequest = await gh.repos.getContent({ + owner: repo.owner, + path: "package.json", + repo: repo.name, + }).catch(() => { + return null; + }); + if (!fileRequest) { + console.log(`Package.json not found in ${repo.owner}/${repo.name}`); + continue; + } + const file = fileRequest.data; + if (!("type" in file) || file.type !== "file") { + console.log(`Package.json found but is not file.`); + continue; + } + const { content } = file; + const parsed = Buffer.from(content, "base64").toString(); + const serialised: { + dependencies?: Record; + devDependencies?: Record; + } | null = serialiseJsonOrError(parsed); + if (!serialised) { + console.log(`Failed to serialise ${parsed}`); + continue; + } + const deps: Record = {}; + if (serialised.dependencies) { + Object.assign(deps, serialised.dependencies); + } + if (serialised.devDependencies) { + Object.assign(deps, serialised.devDependencies); + } + + console.log(`Auditing packages in ${repo.owner}/${repo.name}...`); + + for (const dep of vulnerablePackages) { + if (!(dep.name in deps)) { + continue; + } + if (dep.version !== deps[dep.name]) { + console.log( + `Found ${dep.name}: ${dep.version} but it was not the vulnerable ${String(deps[dep.name])} version.`, + ); + await appendFile( + resultPath, + `${repo.owner}/${repo.name}: Found ${dep.name} but ${String(deps[dep.name])} is not the vulnerable ${dep.version} version.\n`, + ); + continue; + } + console.log( + `FOUND VULNERABLE ${dep.name}: ${dep.version} IN ${repo.owner}/${repo.name}!!!!`, + ); + await appendFile( + resultPath, + `!! FOUND VULNERALBE ${dep.name}: ${dep.version} IN ${repo.owner}/${repo.name} !!\n`, + ); + } +} + +console.log("All done!"); diff --git a/src/utils/serialiseJsonOrError.ts b/src/utils/serialiseJsonOrError.ts new file mode 100644 index 0000000..eb805db --- /dev/null +++ b/src/utils/serialiseJsonOrError.ts @@ -0,0 +1,21 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +/** + * Attempts to serialise a string as JSON. Includes error handling if the + * string is not serialisable. + * @param text -- The text to serialise. + * @returns The serialised object, or null on error. + */ +export const serialiseJsonOrError + = (text: string): Record | null => { + try { + const object = JSON.parse(text); + return object; + } catch { + return null; + } + };