feat: more s3 scripts

src/s3/bulkUpload.ts

@@ -0,0 +1,213 @@
+/**
+ * @copyright NHCarrigan
+ * @license Naomi's Public License
+ * @author Naomi Carrigan
+ */
+import { readFile, readdir } from "node:fs/promises";
+import { join, relative } from "node:path";
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+import { confirm } from "@inquirer/prompts";
+import { SingleBar, Presets } from "cli-progress";
+
+const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
+const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
+
+if (accessKeyId === undefined || secretAccessKey === undefined) {
+  throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
+}
+
+const dataDirectory = join(import.meta.dirname, "..", "..", "data");
+
+/**
+ * Recursively gets all files in a directory.
+ * @param directory - The directory to scan.
+ * @param baseDirectory - The base directory for relative paths.
+ * @returns An array of file paths relative to baseDirectory.
+ */
+const getAllFiles = async(
+  directory: string,
+  baseDirectory: string,
+): Promise<Array<string>> => {
+  const files: Array<string> = [];
+  const entries = await readdir(directory, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(directory, entry.name);
+    const relativePath = relative(baseDirectory, fullPath);
+
+    if (entry.isDirectory()) {
+      const subFiles = await getAllFiles(fullPath, baseDirectory);
+      files.push(...subFiles);
+    } else if (entry.isFile()) {
+      files.push(relativePath);
+    }
+  }
+
+  return files;
+};
+
+/**
+ * Type guard to check if a value is a record.
+ * @param value - The value to check.
+ * @returns Whether the value is a record.
+ */
+const isRecord = (value: unknown): value is Record<string, unknown> => {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+};
+
+/**
+ * Formats a tree node into a string representation.
+ * @param node - The tree node to format.
+ * @param prefix - The prefix for the current level.
+ * @param _isLast - Whether this is the last entry (unused but kept for API consistency).
+ * @returns The formatted tree string.
+ */
+const formatTree = (
+  node: Record<string, unknown>,
+  prefix = "",
+  _isLast = true,
+): string => {
+  const entries = Object.entries(node).sort(([ a ], [ b ]) => {
+    const aIsDirectory = typeof node[a] === "object" && node[a] !== null;
+    const bIsDirectory = typeof node[b] === "object" && node[b] !== null;
+
+    // Directories come first
+    if (aIsDirectory && !bIsDirectory) {
+      return -1;
+    }
+    if (!aIsDirectory && bIsDirectory) {
+      return 1;
+    }
+    return a.localeCompare(b);
+  });
+
+  let result = "";
+
+  for (let index = 0; index < entries.length; index = index + 1) {
+    const entry = entries[index];
+    if (entry === undefined) {
+      continue;
+    }
+    const [ name, value ] = entry;
+    const isLastEntry = index === entries.length - 1;
+    const connector = isLastEntry
+      ? "└── "
+      : "├── ";
+    const nextPrefix = isLastEntry
+      ? "    "
+      : "│   ";
+
+    result = `${result}${prefix}${connector}${name}\n`;
+
+    if (isRecord(value)) {
+      const subTree = formatTree(
+        value,
+        `${prefix}${nextPrefix}`,
+        isLastEntry,
+      );
+      result = `${result}${subTree}`;
+    }
+  }
+
+  return result;
+};
+
+/**
+ * Builds a tree structure from file paths.
+ * @param files - Array of relative file paths.
+ * @returns A tree structure as a string.
+ */
+const buildFileTree = (files: Array<string>): string => {
+  const tree: Record<string, unknown> = {};
+
+  for (const file of files) {
+    const parts = file.split("/");
+    let current = tree;
+
+    for (let index = 0; index < parts.length; index = index + 1) {
+      const part = parts[index];
+      if (part === undefined) {
+        continue;
+      }
+      if (index === parts.length - 1) {
+        // Last part is a file
+        current[part] = null;
+      } else {
+        // It's a directory
+        if (!(part in current) || typeof current[part] !== "object") {
+          current[part] = {};
+        }
+        const currentValue = current[part];
+        if (isRecord(currentValue)) {
+          current = currentValue;
+        }
+      }
+    }
+  }
+
+  return formatTree(tree);
+};
+
+const files = await getAllFiles(dataDirectory, dataDirectory);
+
+if (files.length === 0) {
+  console.log("No files found in the data directory.");
+  process.exit(0);
+}
+
+console.log(`Found ${files.length.toString()} file(s) to upload:\n`);
+console.log(buildFileTree(files));
+console.log(`\nTotal: ${files.length.toString()} file(s)\n`);
+
+const shouldProceed = await confirm({
+  default: false,
+  message: "Do you want to proceed with uploading all these files?",
+});
+
+if (!shouldProceed) {
+  console.log("Upload cancelled.");
+  process.exit(0);
+}
+
+const s3 = new S3Client({
+  credentials: { accessKeyId, secretAccessKey },
+  endpoint:    "https://hel1.your-objectstorage.com",
+  region:      "hel1",
+});
+
+const bar = new SingleBar({}, Presets.shades_classic);
+bar.start(files.length, 0);
+
+let successCount = 0;
+let errorCount = 0;
+
+for (const file of files) {
+  try {
+    const filePath = join(dataDirectory, file);
+    const fileContent = await readFile(filePath);
+
+    const command = new PutObjectCommand({
+      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
+      Body:   fileContent,
+      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
+      Bucket: "nhcarrigan",
+      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
+      Key:    file,
+    });
+
+    await s3.send(command);
+    successCount = successCount + 1;
+  } catch (error) {
+    console.error(`\nError uploading ${file}:`, error);
+    errorCount = errorCount + 1;
+  }
+
+  bar.increment();
+}
+
+bar.stop();
+
+console.log(
+  `\nUpload complete! ${successCount.toString()} succeeded, ${errorCount.toString()} failed.`,
+);
+