feat: add npm audit script, crowdin helpers now write data to disk
Node.js CI / Lint and Test (push) Failing after 34s

This commit is contained in:
2025-09-08 16:55:45 -07:00
parent 3541fdc411
commit b8d8ad35f9
11 changed files with 522 additions and 11 deletions
+3
View File
@@ -2,3 +2,6 @@
CROWDIN_PROJECT_ID="op://Environment Variables - Development/Scripts/Crowdin Project ID"
CROWDIN_API_URL="op://Environment Variables - Development/Scripts/Crowdin API Url"
CROWDIN_TOKEN="op://Environment Variables - Development/Scripts/Crowdin Token"
# Github
GITHUB_TOKEN="op://Environment Variables - Development/Scripts/GitHub Token"
+2
View File
@@ -1,2 +1,4 @@
node_modules
prod
data
!data/.gitkeep
+3
View File
@@ -21,5 +21,8 @@
"eslint": "9.34.0",
"tsx": "4.20.5",
"typescript": "5.9.2"
},
"dependencies": {
"@octokit/rest": "22.0.0"
}
}
+133
View File
@@ -7,6 +7,10 @@ settings:
importers:
.:
dependencies:
'@octokit/rest':
specifier: 22.0.0
version: 22.0.0
devDependencies:
'@nhcarrigan/eslint-config':
specifier: 5.2.0
@@ -309,6 +313,58 @@ packages:
resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
engines: {node: '>= 8'}
'@octokit/auth-token@6.0.0':
resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==}
engines: {node: '>= 20'}
'@octokit/core@7.0.3':
resolution: {integrity: sha512-oNXsh2ywth5aowwIa7RKtawnkdH6LgU1ztfP9AIUCQCvzysB+WeU8o2kyyosDPwBZutPpjZDKPQGIzzrfTWweQ==}
engines: {node: '>= 20'}
'@octokit/endpoint@11.0.0':
resolution: {integrity: sha512-hoYicJZaqISMAI3JfaDr1qMNi48OctWuOih1m80bkYow/ayPw6Jj52tqWJ6GEoFTk1gBqfanSoI1iY99Z5+ekQ==}
engines: {node: '>= 20'}
'@octokit/graphql@9.0.1':
resolution: {integrity: sha512-j1nQNU1ZxNFx2ZtKmL4sMrs4egy5h65OMDmSbVyuCzjOcwsHq6EaYjOTGXPQxgfiN8dJ4CriYHk6zF050WEULg==}
engines: {node: '>= 20'}
'@octokit/openapi-types@25.1.0':
resolution: {integrity: sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA==}
'@octokit/plugin-paginate-rest@13.1.1':
resolution: {integrity: sha512-q9iQGlZlxAVNRN2jDNskJW/Cafy7/XE52wjZ5TTvyhyOD904Cvx//DNyoO3J/MXJ0ve3rPoNWKEg5iZrisQSuw==}
engines: {node: '>= 20'}
peerDependencies:
'@octokit/core': '>=6'
'@octokit/plugin-request-log@6.0.0':
resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==}
engines: {node: '>= 20'}
peerDependencies:
'@octokit/core': '>=6'
'@octokit/plugin-rest-endpoint-methods@16.0.0':
resolution: {integrity: sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g==}
engines: {node: '>= 20'}
peerDependencies:
'@octokit/core': '>=6'
'@octokit/request-error@7.0.0':
resolution: {integrity: sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg==}
engines: {node: '>= 20'}
'@octokit/request@10.0.3':
resolution: {integrity: sha512-V6jhKokg35vk098iBqp2FBKunk3kMTXlmq+PtbV9Gl3TfskWlebSofU9uunVKhUN7xl+0+i5vt0TGTG8/p/7HA==}
engines: {node: '>= 20'}
'@octokit/rest@22.0.0':
resolution: {integrity: sha512-z6tmTu9BTnw51jYGulxrlernpsQYXpui1RK21vmXn8yF5bp6iX16yfTtJYGK5Mh1qDkvDOmp2n8sRMcQmR8jiA==}
engines: {node: '>= 20'}
'@octokit/types@14.1.0':
resolution: {integrity: sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g==}
'@pkgr/core@0.1.2':
resolution: {integrity: sha512-fdDH1LSGfZdTH2sxdpVMw31BanV28K/Gry0cVFxaNP77neJSkd82mM8ErPNYs9e+0O7SdHBLTDzDgwUuy18RnQ==}
engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0}
@@ -676,6 +732,9 @@ packages:
balanced-match@1.0.2:
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
before-after-hook@4.0.0:
resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==}
brace-expansion@1.1.12:
resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}
@@ -998,6 +1057,9 @@ packages:
resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==}
engines: {node: '>=12.0.0'}
fast-content-type-parse@3.0.0:
resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==}
fast-deep-equal@3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
@@ -1847,6 +1909,9 @@ packages:
undici-types@7.10.0:
resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==}
universal-user-agent@7.0.3:
resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==}
update-browserslist-db@1.1.3:
resolution: {integrity: sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==}
hasBin: true
@@ -2190,6 +2255,68 @@ snapshots:
'@nodelib/fs.scandir': 2.1.5
fastq: 1.19.1
'@octokit/auth-token@6.0.0': {}
'@octokit/core@7.0.3':
dependencies:
'@octokit/auth-token': 6.0.0
'@octokit/graphql': 9.0.1
'@octokit/request': 10.0.3
'@octokit/request-error': 7.0.0
'@octokit/types': 14.1.0
before-after-hook: 4.0.0
universal-user-agent: 7.0.3
'@octokit/endpoint@11.0.0':
dependencies:
'@octokit/types': 14.1.0
universal-user-agent: 7.0.3
'@octokit/graphql@9.0.1':
dependencies:
'@octokit/request': 10.0.3
'@octokit/types': 14.1.0
universal-user-agent: 7.0.3
'@octokit/openapi-types@25.1.0': {}
'@octokit/plugin-paginate-rest@13.1.1(@octokit/core@7.0.3)':
dependencies:
'@octokit/core': 7.0.3
'@octokit/types': 14.1.0
'@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.3)':
dependencies:
'@octokit/core': 7.0.3
'@octokit/plugin-rest-endpoint-methods@16.0.0(@octokit/core@7.0.3)':
dependencies:
'@octokit/core': 7.0.3
'@octokit/types': 14.1.0
'@octokit/request-error@7.0.0':
dependencies:
'@octokit/types': 14.1.0
'@octokit/request@10.0.3':
dependencies:
'@octokit/endpoint': 11.0.0
'@octokit/request-error': 7.0.0
'@octokit/types': 14.1.0
fast-content-type-parse: 3.0.0
universal-user-agent: 7.0.3
'@octokit/rest@22.0.0':
dependencies:
'@octokit/core': 7.0.3
'@octokit/plugin-paginate-rest': 13.1.1(@octokit/core@7.0.3)
'@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.3)
'@octokit/plugin-rest-endpoint-methods': 16.0.0(@octokit/core@7.0.3)
'@octokit/types@14.1.0':
dependencies:
'@octokit/openapi-types': 25.1.0
'@pkgr/core@0.1.2': {}
'@rollup/rollup-android-arm-eabi@4.49.0':
@@ -2607,6 +2734,8 @@ snapshots:
balanced-match@1.0.2: {}
before-after-hook@4.0.0: {}
brace-expansion@1.1.12:
dependencies:
balanced-match: 1.0.2
@@ -3102,6 +3231,8 @@ snapshots:
expect-type@1.2.2: {}
fast-content-type-parse@3.0.0: {}
fast-deep-equal@3.1.3: {}
fast-glob@3.3.3:
@@ -4014,6 +4145,8 @@ snapshots:
undici-types@7.10.0: {}
universal-user-agent@7.0.3: {}
update-browserslist-db@1.1.3(browserslist@4.25.3):
dependencies:
browserslist: 4.25.3
+75
View File
@@ -0,0 +1,75 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
import { readFile, appendFile } from "node:fs/promises";
import { join } from "node:path";
import { getLanguages } from "./utils/getLanguages.js";
import type { String } from "./interfaces/string.js";
const projectId = process.env.CROWDIN_PROJECT_ID;
const apiUrl = process.env.CROWDIN_API_URL;
const token = process.env.CROWDIN_TOKEN;
if (
projectId === undefined
|| projectId === ""
|| apiUrl === undefined
|| apiUrl === ""
|| token === undefined
|| token === ""
) {
throw new Error(`Project ID or API URL is missing! Did you run this script with 'op run'?`);
}
const logPath
= join(import.meta.dirname, "..", "..", "data", "crowdin-strings-hidden.txt");
const languages = await getLanguages(projectId, apiUrl, token);
console.log(`Found ${languages.length.toString()} active languages.`);
const rawStrings = await readFile(
join(import.meta.dirname, "..", "..", "data", "crowdin-strings.json"),
"utf-8",
);
const strings: Array<String["data"][0]["data"]> = JSON.parse(rawStrings);
console.log(`Found ${strings.length.toString()} strings.`);
const log = await readFile(
logPath,
"utf-8",
);
const processedIds = log.split("\n");
const unprocessedStrings = strings.filter((string) => {
return !processedIds.includes(string.id.toString());
});
const hidden = unprocessedStrings.filter((string) => {
return string.isHidden;
});
console.log(`Processing ${hidden.length.toString()} hidden strings.`);
for (const string of hidden) {
console.log(`Deleting translations for ${string.id.toString()}`);
await Promise.all(languages.map(async(language) => {
await fetch(`${apiUrl}/projects/${projectId}/translations`, {
body: JSON.stringify({
languageId: language,
stringId: string.id,
}),
headers: {
"authorization": `Bearer ${token}`,
// eslint-disable-next-line @typescript-eslint/naming-convention -- header.
"content-type": "application/json",
},
method: "DELETE",
});
}));
await appendFile(logPath, `${string.id.toString()}\n`);
}
console.log("Completed!");
+36
View File
@@ -0,0 +1,36 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
export interface String {
data: Array<{
data: {
id: number;
projectId: number;
branchId: number;
identifier: string;
text: string;
type: string;
context: string;
maxLength: number;
isHidden: boolean;
isDuplicate: boolean;
masterStringId: number;
hasPlurals: boolean;
isIcu: boolean;
labelIds: Array<number>;
webUrl: string;
createdAt: string;
updatedAt: string;
fileId: number;
directoryId: number;
revision: number;
};
}>;
pagination: {
offset: number;
limit: number;
};
}
+5 -9
View File
@@ -18,10 +18,10 @@ export const getFiles
projectId: string,
apiUrl: string,
token: string,
): Promise<Array<number>> => {
): Promise<Array<File["data"][0]>> => {
const url = `${apiUrl}/projects/${projectId}/files?limit=500`;
let offset = 0;
const ids: Array<number> = [];
const files: Array<File["data"][0]> = [];
console.log(`Requesting files ${offset.toString()} to ${(offset + 500).toString()}`);
let request = await fetch(url, {
@@ -30,9 +30,7 @@ export const getFiles
},
});
let response: File = await request.json();
ids.push(...response.data.map((datum) => {
return datum.data.id;
}));
files.push(...response.data);
while (response.data.length >= 500) {
offset = offset + 500;
console.log(`Requesting files ${offset.toString()} to ${(offset + 500).toString()}`);
@@ -42,9 +40,7 @@ export const getFiles
},
});
response = await request.json();
ids.push(...response.data.map((datum) => {
return datum.data.id;
}));
files.push(...response.data);
}
return ids;
return files;
};
+50
View File
@@ -0,0 +1,50 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
import type { String } from "../interfaces/string.js";
/**
* Gets a list of all strings from a project.
* @param projectId - The project ID (a numeric string).
* @param apiUrl - The base URL for the API.
* @param token - The API key.
* @returns An array of string objects.
*/
export const getStrings
= async(
projectId: string,
apiUrl: string,
token: string,
): Promise<Array<String["data"][0]["data"]>> => {
const url = `${apiUrl}/projects/${projectId}/strings?limit=500`;
let offset = 0;
const strings: Array<String["data"][0]["data"]> = [];
console.log(`Requesting strings ${offset.toString()} to ${(offset + 500).toString()}`);
let request = await fetch(url, {
headers: {
authorization: `Bearer ${token}`,
},
});
let response: String = await request.json();
strings.push(...response.data.map((datum) => {
return datum.data;
}));
while (response.data.length >= 500) {
offset = offset + 500;
console.log(`Requesting strings ${offset.toString()} to ${(offset + 500).toString()}`);
request = await fetch(`${url}&offset=${offset.toString()}`, {
headers: {
authorization: `Bearer ${token}`,
},
});
response = await request.json();
strings.push(...response.data.map((datum) => {
return datum.data;
}));
}
return strings;
};
+41
View File
@@ -0,0 +1,41 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { getFiles } from "./utils/getFiles.js";
import { getStrings } from "./utils/getStrings.js";
const projectId = process.env.CROWDIN_PROJECT_ID;
const apiUrl = process.env.CROWDIN_API_URL;
const token = process.env.CROWDIN_TOKEN;
if (
projectId === undefined
|| projectId === ""
|| apiUrl === undefined
|| apiUrl === ""
|| token === undefined
|| token === ""
) {
throw new Error(`Project ID or API URL is missing! Did you run this script with 'op run'?`);
}
const files = await getFiles(projectId, apiUrl, token);
const strings = await getStrings(projectId, apiUrl, token);
await writeFile(
join(import.meta.dirname, "..", "..", "data", "crowdin-files.json"),
JSON.stringify(files, null, 2),
"utf-8",
);
await writeFile(
join(import.meta.dirname, "..", "..", "data", "crowdin-strings.json"),
JSON.stringify(strings, null, 2),
"utf-8",
);
console.log("Loaded files and strings!");
+151
View File
@@ -0,0 +1,151 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
import { writeFile, appendFile } from "node:fs/promises";
import { join } from "node:path";
import { Octokit } from "@octokit/rest";
import { serialiseJsonOrError } from "../utils/serialiseJsonOrError.js";
if (process.env.GITHUB_TOKEN === undefined) {
throw new Error("Missing Github Token - did you run this with `op`?");
}
const resultPath = join(
import.meta.dirname,
"..",
"..",
"data",
"npm-vulnerabilities.txt",
);
const orgsToCheck = [
"deepgram",
"deepgram-devs",
"deepgram-starters",
];
const vulnerablePackages: Array<{ name: string; version: string }> = [
{ name: "ansi-regex", version: "6.2.1" },
{ name: "ansi-styles", version: "6.2.2" },
{ name: "backslash", version: "0.2.1" },
{ name: "chalk-template", version: "1.1.1" },
{ name: "chalk", version: "5.6.1" },
{ name: "color-convert", version: "3.1.1" },
{ name: "color-name", version: "2.0.1" },
{ name: "color-string", version: "2.1.1" },
{ name: "color", version: "5.0.1" },
{ name: "debug", version: "4.4.2" },
{ name: "has-ansi", version: "6.0.1" },
{ name: "is-arrayish", version: "0.3.3" },
{ name: "simple-swizzle", version: "0.2.3" },
{ name: "slice-ansi", version: "7.1.1" },
{ name: "strip-ansi", version: "7.1.1" },
{ name: "supports-color", version: "10.2.1" },
{ name: "supports-hyperlinks", version: "4.1.1" },
{ name: "wrap-ansi", version: "9.0.1" },
];
const gh = new Octokit({
auth: process.env.GITHUB_TOKEN,
});
const repositories: Array<{ name: string; owner: string }>
= [];
await writeFile(resultPath, "", "utf-8");
for (const org of orgsToCheck) {
let page = 1;
console.log(`Fetching ${org} repositories page ${page.toString()}`);
let repos = await gh.repos.listForOrg(
{
org: org,
// eslint-disable-next-line @typescript-eslint/naming-convention -- SDK signature.
per_page: 100,
},
);
repositories.push(...repos.data.map((repo) => {
return { name: repo.name, owner: org };
}));
while (repos.data.length >= 100) {
page = page + 1;
console.log(`Fetching ${org} repositories page ${page.toString()}`);
repos = await gh.repos.listForOrg({
org: org,
page: page,
// eslint-disable-next-line @typescript-eslint/naming-convention -- SDK signature.
per_page: 100,
});
repositories.push(...repos.data.map((repo) => {
return { name: repo.name, owner: org };
}));
}
}
console.log(`Found ${repositories.length.toString()} repositories in ${orgsToCheck.length.toString()} orgs.`);
for (const repo of repositories) {
const fileRequest = await gh.repos.getContent({
owner: repo.owner,
path: "package.json",
repo: repo.name,
}).catch(() => {
return null;
});
if (!fileRequest) {
console.log(`Package.json not found in ${repo.owner}/${repo.name}`);
continue;
}
const file = fileRequest.data;
if (!("type" in file) || file.type !== "file") {
console.log(`Package.json found but is not file.`);
continue;
}
const { content } = file;
const parsed = Buffer.from(content, "base64").toString();
const serialised: {
dependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
} | null = serialiseJsonOrError(parsed);
if (!serialised) {
console.log(`Failed to serialise ${parsed}`);
continue;
}
const deps: Record<string, string> = {};
if (serialised.dependencies) {
Object.assign(deps, serialised.dependencies);
}
if (serialised.devDependencies) {
Object.assign(deps, serialised.devDependencies);
}
console.log(`Auditing packages in ${repo.owner}/${repo.name}...`);
for (const dep of vulnerablePackages) {
if (!(dep.name in deps)) {
continue;
}
if (dep.version !== deps[dep.name]) {
console.log(
`Found ${dep.name}: ${dep.version} but it was not the vulnerable ${String(deps[dep.name])} version.`,
);
await appendFile(
resultPath,
`${repo.owner}/${repo.name}: Found ${dep.name} but ${String(deps[dep.name])} is not the vulnerable ${dep.version} version.\n`,
);
continue;
}
console.log(
`FOUND VULNERABLE ${dep.name}: ${dep.version} IN ${repo.owner}/${repo.name}!!!!`,
);
await appendFile(
resultPath,
`!! FOUND VULNERALBE ${dep.name}: ${dep.version} IN ${repo.owner}/${repo.name} !!\n`,
);
}
}
console.log("All done!");
+21
View File
@@ -0,0 +1,21 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
/**
* Attempts to serialise a string as JSON. Includes error handling if the
* string is not serialisable.
* @param text -- The text to serialise.
* @returns The serialised object, or null on error.
*/
export const serialiseJsonOrError
= (text: string): Record<string, unknown> | null => {
try {
const object = JSON.parse(text);
return object;
} catch {
return null;
}
};