feat: ability to delete s3 objects, make endpoint an env #2

Merged
naomi merged 2 commits from feat/s3 into main 2026-02-02 10:58:45 -08:00
7 changed files with 199 additions and 5 deletions
+3
View File
@@ -45,6 +45,9 @@ lint-ts:
lint-py:
cd python && uv run ruff check .
format-ts:
cd typescript && pnpm exec eslint src --fix
# Format Python code
format: format-py
+3 -2
View File
@@ -10,11 +10,12 @@ GITHUB_TOKEN="op://Environment Variables - Development/Ephemere/GitHub Token"
DISCORD_TOKEN="op://Environment Variables - Development/Ephemere/Discord Token"
DISCORD_CLIENT_ID="op://Private/Guild Counter/client id"
DISCORD_CLIENT_SECRET="op://Private/Guild Counter/client secret"
DISCORD_BOT_TOKEN="op://Private/Amari Bot/Token"
DISCORD_BOT_TOKEN="op://Environment Variables - Naomi/Amari/bot token"
# AWS
AWS_ACCESS_KEY_ID="op://Private/Hetzner/S3 Access Key ID"
AWS_SECRET_ACCESS_KEY="op://Private/Hetzner/S3 Secret Access Key"
S3_ENDPOINT="op://Private/Hetzner/S3 Endpoint"
# Gitea
GITEA_TOKEN="op://Private/Gitea/token"
@@ -25,4 +26,4 @@ DOJO_TOKEN="op://Private/DefectDojo/token"
# Discourse
DISCOURSE_URL="op://Environment Variables - Development/Ephemere/Discourse URL"
DISCOURSE_API_KEY="op://Environment Variables - Development/Ephemere/Discourse Key"
DISCOURSE_API_USERNAME="op://Environment Variables - Development/Ephemere/Discourse Username"
DISCOURSE_API_USERNAME="op://Environment Variables - Development/Ephemere/Discourse Username"
+3
View File
@@ -17,6 +17,9 @@ build:
lint:
pnpm exec eslint src --max-warnings 0
format:
pnpm exec eslint src --fix
test:
@echo "No tests configured yet"
@exit 0
+6 -1
View File
@@ -12,11 +12,16 @@ import { getMimeType } from "../utils/mimeType.js";
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
const endpoint = process.env.S3_ENDPOINT;
if (accessKeyId === undefined || secretAccessKey === undefined) {
throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
}
if (endpoint === undefined) {
throw new Error("S3_ENDPOINT is not set");
}
const dataDirectory = join(import.meta.dirname, "..", "..", "data");
/**
@@ -172,7 +177,7 @@ if (!shouldProceed) {
const s3 = new S3Client({
credentials: { accessKeyId, secretAccessKey },
endpoint: "https://hel1.your-objectstorage.com",
endpoint: endpoint,
region: "hel1",
});
+6 -1
View File
@@ -15,6 +15,7 @@ import { getMimeType } from "../utils/mimeType.js";
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
const endpoint = process.env.S3_ENDPOINT;
if (accessKeyId === undefined || secretAccessKey === undefined) {
throw new Error(
@@ -22,9 +23,13 @@ if (accessKeyId === undefined || secretAccessKey === undefined) {
);
}
if (endpoint === undefined) {
throw new Error("S3_ENDPOINT is not set");
}
const s3 = new S3Client({
credentials: { accessKeyId, secretAccessKey },
endpoint: "https://hel1.your-objectstorage.com",
endpoint: endpoint,
region: "hel1",
});
+172
View File
@@ -0,0 +1,172 @@
/**
* @copyright NHCarrigan
* @license Naomi's Public License
* @author Naomi Carrigan
*/
import {
S3Client, ListObjectsV2Command, DeleteObjectsCommand,
type ListObjectsV2CommandOutput,
} from "@aws-sdk/client-s3";
import { input, confirm } from "@inquirer/prompts";
import { SingleBar, Presets } from "cli-progress";
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
const endpoint = process.env.S3_ENDPOINT;
if (accessKeyId === undefined || secretAccessKey === undefined) {
throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
}
if (endpoint === undefined) {
throw new Error("S3_ENDPOINT is not set");
}
// Get bucket name
const bucketName = await input({
message: "Enter the S3 bucket name:",
validate: (value) => {
if (value.trim() === "") {
return "Bucket name cannot be empty";
}
// Basic S3 bucket name validation
if (!/^[\d.a-z-]+$/.test(value)) {
return `Bucket name can only contain lowercase letters, numbers, dots, and hyphens`;
}
return true;
},
});
// Create S3 client
const s3 = new S3Client({
credentials: { accessKeyId, secretAccessKey },
endpoint: endpoint,
region: "hel1",
});
// First, count total objects to delete
console.log("\nCounting objects in bucket...");
let totalObjects = 0;
let continuationToken: string | null = null;
do {
const listCommand = new ListObjectsV2Command({
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
Bucket: bucketName,
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
ContinuationToken: continuationToken ?? undefined,
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
MaxKeys: 1000,
});
const listResponse: ListObjectsV2CommandOutput = await s3.send(listCommand);
totalObjects = totalObjects + (listResponse.Contents?.length ?? 0);
continuationToken = listResponse.NextContinuationToken ?? null;
} while (continuationToken !== null);
if (totalObjects === 0) {
console.log("✨ No files found in the bucket!");
process.exit(0);
}
console.log(`\nFound ${totalObjects.toString()} object(s) to delete.`);
// Safety confirmation
console.log(`\n⚠️ WARNING: This will DELETE ALL ${totalObjects.toString()} FILES in the bucket "${bucketName}"`);
console.log("This action cannot be undone!\n");
// First confirmation - type bucket name
await input({
message: `Type the bucket name "${bucketName}" to confirm deletion:`,
validate: (value) => {
if (value !== bucketName) {
return `Please type "${bucketName}" exactly to confirm`;
}
return true;
},
});
// Second confirmation - yes/no
const finalConfirm = await confirm({
default: false,
message: "Are you ABSOLUTELY sure you want to delete all files?",
});
if (!finalConfirm) {
console.log("❌ Operation cancelled. No files were deleted.");
process.exit(0);
}
console.log("\n🚀 Starting deletion process...\n");
// Initialize progress bar
const bar = new SingleBar({}, Presets.shades_classic);
bar.start(totalObjects, 0);
let successCount = 0;
let errorCount = 0;
continuationToken = null;
do {
// List objects in the bucket
const listCommand = new ListObjectsV2Command({
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
Bucket: bucketName,
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
ContinuationToken: continuationToken ?? undefined,
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
MaxKeys: 1000,
});
const listResponse: ListObjectsV2CommandOutput = await s3.send(listCommand);
if (!listResponse.Contents || listResponse.Contents.length === 0) {
break;
}
// Prepare objects for deletion
const objectsToDelete = listResponse.Contents.
filter((object) => {
return object.Key !== undefined;
}).
map((object) => {
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
return { Key: object.Key };
});
// Delete objects in batch
const deleteCommand = new DeleteObjectsCommand({
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
Bucket: bucketName,
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
Delete: {
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
Objects: objectsToDelete,
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
Quiet: false,
},
});
const deleteResponse = await s3.send(deleteCommand);
const deletedCount = deleteResponse.Deleted?.length ?? 0;
successCount = successCount + deletedCount;
// Check for errors
if (deleteResponse.Errors && deleteResponse.Errors.length > 0) {
errorCount = errorCount + deleteResponse.Errors.length;
for (const error of deleteResponse.Errors) {
console.error(`\n⚠️ Error deleting ${error.Key ?? ""}: ${error.Message ?? ""}`);
}
}
bar.increment(deletedCount);
continuationToken = listResponse.NextContinuationToken ?? null;
} while (continuationToken !== null);
bar.stop();
console.log(
`\n✅ Delete complete! ${successCount.toString()} succeeded, ${errorCount.toString()} failed.`,
);
+6 -1
View File
@@ -11,11 +11,16 @@ import { getMimeType } from "../utils/mimeType.js";
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
const endpoint = process.env.S3_ENDPOINT;
if (accessKeyId === undefined || secretAccessKey === undefined) {
throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
}
if (endpoint === undefined) {
throw new Error("S3_ENDPOINT is not set");
}
const fileName = await input({
message:
// eslint-disable-next-line stylistic/max-len -- Big boi string.
@@ -40,7 +45,7 @@ if (uploadPath === "") {
const s3 = new S3Client({
credentials: { accessKeyId, secretAccessKey },
endpoint: "https://hel1.your-objectstorage.com",
endpoint: endpoint,
region: "hel1",
});