ephemere/src/s3/bulkUpload.ts

/**
 * @copyright NHCarrigan
 * @license Naomi's Public License
 * @author Naomi Carrigan
 */
import { readFile, readdir } from "node:fs/promises";
import { join, relative } from "node:path";
import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
import { confirm } from "@inquirer/prompts";
import { SingleBar, Presets } from "cli-progress";
import { getMimeType } from "../utils/mimeType.js";

const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;

if (accessKeyId === undefined || secretAccessKey === undefined) {
  throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
}

const dataDirectory = join(import.meta.dirname, "..", "..", "data");

/**
 * Recursively gets all files in a directory.
 * @param directory - The directory to scan.
 * @param baseDirectory - The base directory for relative paths.
 * @returns An array of file paths relative to baseDirectory.
 */
const getAllFiles = async(
  directory: string,
  baseDirectory: string,
): Promise<Array<string>> => {
  const files: Array<string> = [];
  const entries = await readdir(directory, { withFileTypes: true });

  for (const entry of entries) {
    const fullPath = join(directory, entry.name);
    const relativePath = relative(baseDirectory, fullPath);

    if (entry.isDirectory()) {
      const subFiles = await getAllFiles(fullPath, baseDirectory);
      files.push(...subFiles);
    } else if (entry.isFile()) {
      files.push(relativePath);
    }
  }

  return files;
};

/**
 * Type guard to check if a value is a record.
 * @param value - The value to check.
 * @returns Whether the value is a record.
 */
const isRecord = (value: unknown): value is Record<string, unknown> => {
  return typeof value === "object" && value !== null && !Array.isArray(value);
};

/**
 * Formats a tree node into a string representation.
 * @param node - The tree node to format.
 * @param prefix - The prefix for the current level.
 * @param _isLast - Whether this is the last entry (unused but kept for API consistency).
 * @returns The formatted tree string.
 */
const formatTree = (
  node: Record<string, unknown>,
  prefix = "",
  _isLast = true,
): string => {
  const entries = Object.entries(node).sort(([ a ], [ b ]) => {
    const aIsDirectory = typeof node[a] === "object" && node[a] !== null;
    const bIsDirectory = typeof node[b] === "object" && node[b] !== null;

    // Directories come first
    if (aIsDirectory && !bIsDirectory) {
      return -1;
    }
    if (!aIsDirectory && bIsDirectory) {
      return 1;
    }
    return a.localeCompare(b);
  });

  let result = "";

  for (let index = 0; index < entries.length; index = index + 1) {
    const entry = entries[index];
    if (entry === undefined) {
      continue;
    }
    const [ name, value ] = entry;
    const isLastEntry = index === entries.length - 1;
    const connector = isLastEntry
      ? "└── "
      : "├── ";
    const nextPrefix = isLastEntry
      ? "    "
      : "│   ";

    result = `${result}${prefix}${connector}${name}\n`;

    if (isRecord(value)) {
      const subTree = formatTree(
        value,
        `${prefix}${nextPrefix}`,
        isLastEntry,
      );
      result = `${result}${subTree}`;
    }
  }

  return result;
};

/**
 * Builds a tree structure from file paths.
 * @param files - Array of relative file paths.
 * @returns A tree structure as a string.
 */
const buildFileTree = (files: Array<string>): string => {
  const tree: Record<string, unknown> = {};

  for (const file of files) {
    const parts = file.split("/");
    let current = tree;

    for (let index = 0; index < parts.length; index = index + 1) {
      const part = parts[index];
      if (part === undefined) {
        continue;
      }
      if (index === parts.length - 1) {
        // Last part is a file
        current[part] = null;
      } else {
        // It's a directory
        if (!(part in current) || typeof current[part] !== "object") {
          current[part] = {};
        }
        const currentValue = current[part];
        if (isRecord(currentValue)) {
          current = currentValue;
        }
      }
    }
  }

  return formatTree(tree);
};

const files = await getAllFiles(dataDirectory, dataDirectory);

if (files.length === 0) {
  console.log("No files found in the data directory.");
  process.exit(0);
}

console.log(`Found ${files.length.toString()} file(s) to upload:\n`);
console.log(buildFileTree(files));
console.log(`\nTotal: ${files.length.toString()} file(s)\n`);

const shouldProceed = await confirm({
  default: false,
  message: "Do you want to proceed with uploading all these files?",
});

if (!shouldProceed) {
  console.log("Upload cancelled.");
  process.exit(0);
}

const s3 = new S3Client({
  credentials: { accessKeyId, secretAccessKey },
  endpoint:    "https://hel1.your-objectstorage.com",
  region:      "hel1",
});

const bar = new SingleBar({}, Presets.shades_classic);
bar.start(files.length, 0);

let successCount = 0;
let errorCount = 0;

for (const file of files) {
  try {
    const filePath = join(dataDirectory, file);
    const fileContent = await readFile(filePath);
    const contentType = getMimeType(file);

    const command = new PutObjectCommand({
      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
      Body:        fileContent,
      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
      Bucket:      "nhcarrigan",
      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
      ContentType: contentType,
      // eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
      Key:         file,
    });

    await s3.send(command);
    successCount = successCount + 1;
  } catch (error) {
    console.error(`\nError uploading ${file}:`, error);
    errorCount = errorCount + 1;
  }

  bar.increment();
}

bar.stop();

console.log(
  `\nUpload complete! ${successCount.toString()} succeeded, ${errorCount.toString()} failed.`,
);