feat: add npm audit script, crowdin helpers now write data to disk

src/crowdin/clearHiddenTranslations.ts

@@ -0,0 +1,75 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+
+import { readFile, appendFile } from "node:fs/promises";
+import { join } from "node:path";
+import { getLanguages } from "./utils/getLanguages.js";
+import type { String } from "./interfaces/string.js";
+
+const projectId = process.env.CROWDIN_PROJECT_ID;
+const apiUrl = process.env.CROWDIN_API_URL;
+const token = process.env.CROWDIN_TOKEN;
+
+if (
+  projectId === undefined
+    || projectId === ""
+    || apiUrl === undefined
+    || apiUrl === ""
+    || token === undefined
+    || token === ""
+) {
+  throw new Error(`Project ID or API URL is missing! Did you run this script with 'op run'?`);
+}
+
+const logPath
+  = join(import.meta.dirname, "..", "..", "data", "crowdin-strings-hidden.txt");
+
+const languages = await getLanguages(projectId, apiUrl, token);
+console.log(`Found ${languages.length.toString()} active languages.`);
+const rawStrings = await readFile(
+  join(import.meta.dirname, "..", "..", "data", "crowdin-strings.json"),
+  "utf-8",
+);
+const strings: Array<String["data"][0]["data"]> = JSON.parse(rawStrings);
+console.log(`Found ${strings.length.toString()} strings.`);
+
+const log = await readFile(
+  logPath,
+  "utf-8",
+);
+
+const processedIds = log.split("\n");
+
+const unprocessedStrings = strings.filter((string) => {
+  return !processedIds.includes(string.id.toString());
+});
+
+const hidden = unprocessedStrings.filter((string) => {
+  return string.isHidden;
+});
+
+console.log(`Processing ${hidden.length.toString()} hidden strings.`);
+
+for (const string of hidden) {
+  console.log(`Deleting translations for ${string.id.toString()}`);
+  await Promise.all(languages.map(async(language) => {
+    await fetch(`${apiUrl}/projects/${projectId}/translations`, {
+      body: JSON.stringify({
+        languageId: language,
+        stringId:   string.id,
+      }),
+      headers: {
+        "authorization": `Bearer ${token}`,
+        // eslint-disable-next-line @typescript-eslint/naming-convention -- header.
+        "content-type":  "application/json",
+      },
+      method: "DELETE",
+    });
+  }));
+  await appendFile(logPath, `${string.id.toString()}\n`);
+}
+
+console.log("Completed!");

src/crowdin/interfaces/string.ts

@@ -0,0 +1,36 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+
+export interface String {
+  data: Array<{
+    data: {
+      id:             number;
+      projectId:      number;
+      branchId:       number;
+      identifier:     string;
+      text:           string;
+      type:           string;
+      context:        string;
+      maxLength:      number;
+      isHidden:       boolean;
+      isDuplicate:    boolean;
+      masterStringId: number;
+      hasPlurals:     boolean;
+      isIcu:          boolean;
+      labelIds:       Array<number>;
+      webUrl:         string;
+      createdAt:      string;
+      updatedAt:      string;
+      fileId:         number;
+      directoryId:    number;
+      revision:       number;
+    };
+  }>;
+  pagination: {
+    offset: number;
+    limit:  number;
+  };
+}

src/crowdin/utils/getFiles.ts

@@ -18,10 +18,10 @@ export const getFiles
     projectId: string,
     apiUrl: string,
     token: string,
-  ): Promise<Array<number>> => {
+  ): Promise<Array<File["data"][0]>> => {
     const url = `${apiUrl}/projects/${projectId}/files?limit=500`;
     let offset = 0;
-    const ids: Array<number> = [];
+    const files: Array<File["data"][0]> = [];
 
     console.log(`Requesting files ${offset.toString()} to ${(offset + 500).toString()}`);
     let request = await fetch(url, {
@@ -30,9 +30,7 @@ export const getFiles
       },
     });
     let response: File = await request.json();
-    ids.push(...response.data.map((datum) => {
-      return datum.data.id;
-    }));
+    files.push(...response.data);
     while (response.data.length >= 500) {
       offset = offset + 500;
       console.log(`Requesting files ${offset.toString()} to ${(offset + 500).toString()}`);
@@ -42,9 +40,7 @@ export const getFiles
         },
       });
       response = await request.json();
-      ids.push(...response.data.map((datum) => {
-        return datum.data.id;
-      }));
+      files.push(...response.data);
     }
-    return ids;
+    return files;
   };

src/crowdin/utils/getStrings.ts

@@ -0,0 +1,50 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+
+import type { String } from "../interfaces/string.js";
+
+/**
+ * Gets a list of all strings from a project.
+ * @param projectId - The project ID (a numeric string).
+ * @param apiUrl - The base URL for the API.
+ * @param token - The API key.
+ * @returns An array of string objects.
+ */
+export const getStrings
+  = async(
+    projectId: string,
+    apiUrl: string,
+    token: string,
+  ): Promise<Array<String["data"][0]["data"]>> => {
+    const url = `${apiUrl}/projects/${projectId}/strings?limit=500`;
+    let offset = 0;
+    const strings: Array<String["data"][0]["data"]> = [];
+
+    console.log(`Requesting strings ${offset.toString()} to ${(offset + 500).toString()}`);
+    let request = await fetch(url, {
+      headers: {
+        authorization: `Bearer ${token}`,
+      },
+    });
+    let response: String = await request.json();
+    strings.push(...response.data.map((datum) => {
+      return datum.data;
+    }));
+    while (response.data.length >= 500) {
+      offset = offset + 500;
+      console.log(`Requesting strings ${offset.toString()} to ${(offset + 500).toString()}`);
+      request = await fetch(`${url}&offset=${offset.toString()}`, {
+        headers: {
+          authorization: `Bearer ${token}`,
+        },
+      });
+      response = await request.json();
+      strings.push(...response.data.map((datum) => {
+        return datum.data;
+      }));
+    }
+    return strings;
+  };

src/crowdin/writeData.ts

@@ -0,0 +1,41 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { getFiles } from "./utils/getFiles.js";
+import { getStrings } from "./utils/getStrings.js";
+
+const projectId = process.env.CROWDIN_PROJECT_ID;
+const apiUrl = process.env.CROWDIN_API_URL;
+const token = process.env.CROWDIN_TOKEN;
+
+if (
+  projectId === undefined
+    || projectId === ""
+    || apiUrl === undefined
+    || apiUrl === ""
+    || token === undefined
+    || token === ""
+) {
+  throw new Error(`Project ID or API URL is missing! Did you run this script with 'op run'?`);
+}
+
+const files = await getFiles(projectId, apiUrl, token);
+const strings = await getStrings(projectId, apiUrl, token);
+
+await writeFile(
+  join(import.meta.dirname, "..", "..", "data", "crowdin-files.json"),
+  JSON.stringify(files, null, 2),
+  "utf-8",
+);
+await writeFile(
+  join(import.meta.dirname, "..", "..", "data", "crowdin-strings.json"),
+  JSON.stringify(strings, null, 2),
+  "utf-8",
+);
+
+console.log("Loaded files and strings!");

src/github/auditNpmPackages.ts

@@ -0,0 +1,151 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+
+import { writeFile, appendFile } from "node:fs/promises";
+import { join } from "node:path";
+import { Octokit } from "@octokit/rest";
+import { serialiseJsonOrError } from "../utils/serialiseJsonOrError.js";
+
+if (process.env.GITHUB_TOKEN === undefined) {
+  throw new Error("Missing Github Token - did you run this with `op`?");
+}
+
+const resultPath = join(
+  import.meta.dirname,
+  "..",
+  "..",
+  "data",
+  "npm-vulnerabilities.txt",
+);
+
+const orgsToCheck = [
+  "deepgram",
+  "deepgram-devs",
+  "deepgram-starters",
+];
+
+const vulnerablePackages: Array<{ name: string; version: string }> = [
+  { name: "ansi-regex", version: "6.2.1" },
+  { name: "ansi-styles", version: "6.2.2" },
+  { name: "backslash", version: "0.2.1" },
+  { name: "chalk-template", version: "1.1.1" },
+  { name: "chalk", version: "5.6.1" },
+  { name: "color-convert", version: "3.1.1" },
+  { name: "color-name", version: "2.0.1" },
+  { name: "color-string", version: "2.1.1" },
+  { name: "color", version: "5.0.1" },
+  { name: "debug", version: "4.4.2" },
+  { name: "has-ansi", version: "6.0.1" },
+  { name: "is-arrayish", version: "0.3.3" },
+  { name: "simple-swizzle", version: "0.2.3" },
+  { name: "slice-ansi", version: "7.1.1" },
+  { name: "strip-ansi", version: "7.1.1" },
+  { name: "supports-color", version: "10.2.1" },
+  { name: "supports-hyperlinks", version: "4.1.1" },
+  { name: "wrap-ansi", version: "9.0.1" },
+];
+
+const gh = new Octokit({
+  auth: process.env.GITHUB_TOKEN,
+});
+
+const repositories: Array<{ name: string; owner: string }>
+  = [];
+
+await writeFile(resultPath, "", "utf-8");
+
+for (const org of orgsToCheck) {
+  let page = 1;
+  console.log(`Fetching ${org} repositories page ${page.toString()}`);
+  let repos = await gh.repos.listForOrg(
+    {
+      org:      org,
+      // eslint-disable-next-line @typescript-eslint/naming-convention -- SDK signature.
+      per_page: 100,
+    },
+  );
+  repositories.push(...repos.data.map((repo) => {
+    return { name: repo.name, owner: org };
+  }));
+  while (repos.data.length >= 100) {
+    page = page + 1;
+    console.log(`Fetching ${org} repositories page ${page.toString()}`);
+    repos = await gh.repos.listForOrg({
+      org:      org,
+      page:     page,
+      // eslint-disable-next-line @typescript-eslint/naming-convention -- SDK signature.
+      per_page: 100,
+    });
+    repositories.push(...repos.data.map((repo) => {
+      return { name: repo.name, owner: org };
+    }));
+  }
+}
+
+console.log(`Found ${repositories.length.toString()} repositories in ${orgsToCheck.length.toString()} orgs.`);
+
+for (const repo of repositories) {
+  const fileRequest = await gh.repos.getContent({
+    owner: repo.owner,
+    path:  "package.json",
+    repo:  repo.name,
+  }).catch(() => {
+    return null;
+  });
+  if (!fileRequest) {
+    console.log(`Package.json not found in ${repo.owner}/${repo.name}`);
+    continue;
+  }
+  const file = fileRequest.data;
+  if (!("type" in file) || file.type !== "file") {
+    console.log(`Package.json found but is not file.`);
+    continue;
+  }
+  const { content } = file;
+  const parsed = Buffer.from(content, "base64").toString();
+  const serialised: {
+    dependencies?:    Record<string, string>;
+    devDependencies?: Record<string, string>;
+  } | null = serialiseJsonOrError(parsed);
+  if (!serialised) {
+    console.log(`Failed to serialise ${parsed}`);
+    continue;
+  }
+  const deps: Record<string, string> = {};
+  if (serialised.dependencies) {
+    Object.assign(deps, serialised.dependencies);
+  }
+  if (serialised.devDependencies) {
+    Object.assign(deps, serialised.devDependencies);
+  }
+
+  console.log(`Auditing packages in ${repo.owner}/${repo.name}...`);
+
+  for (const dep of vulnerablePackages) {
+    if (!(dep.name in deps)) {
+      continue;
+    }
+    if (dep.version !== deps[dep.name]) {
+      console.log(
+        `Found ${dep.name}: ${dep.version} but it was not the vulnerable ${String(deps[dep.name])} version.`,
+      );
+      await appendFile(
+        resultPath,
+        `${repo.owner}/${repo.name}: Found ${dep.name} but ${String(deps[dep.name])} is not the vulnerable ${dep.version} version.\n`,
+      );
+      continue;
+    }
+    console.log(
+      `FOUND VULNERABLE ${dep.name}: ${dep.version} IN ${repo.owner}/${repo.name}!!!!`,
+    );
+    await appendFile(
+      resultPath,
+      `!! FOUND VULNERALBE ${dep.name}: ${dep.version} IN ${repo.owner}/${repo.name} !!\n`,
+    );
+  }
+}
+
+console.log("All done!");

src/utils/serialiseJsonOrError.ts

@@ -0,0 +1,21 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+
+/**
+ * Attempts to serialise a string as JSON. Includes error handling if the
+ * string is not serialisable.
+ * @param text -- The text to serialise.
+ * @returns The serialised object, or null on error.
+ */
+export const serialiseJsonOrError
+  = (text: string): Record<string, unknown> | null => {
+    try {
+      const object = JSON.parse(text);
+      return object;
+    } catch {
+      return null;
+    }
+  };