generated from nhcarrigan/template
feat: ability to delete s3 objects, make endpoint an env #2
@@ -45,6 +45,9 @@ lint-ts:
|
||||
lint-py:
|
||||
cd python && uv run ruff check .
|
||||
|
||||
format-ts:
|
||||
cd typescript && pnpm exec eslint src --fix
|
||||
|
||||
# Format Python code
|
||||
format: format-py
|
||||
|
||||
|
||||
@@ -10,11 +10,12 @@ GITHUB_TOKEN="op://Environment Variables - Development/Ephemere/GitHub Token"
|
||||
DISCORD_TOKEN="op://Environment Variables - Development/Ephemere/Discord Token"
|
||||
DISCORD_CLIENT_ID="op://Private/Guild Counter/client id"
|
||||
DISCORD_CLIENT_SECRET="op://Private/Guild Counter/client secret"
|
||||
DISCORD_BOT_TOKEN="op://Private/Amari Bot/Token"
|
||||
DISCORD_BOT_TOKEN="op://Environment Variables - Naomi/Amari/bot token"
|
||||
|
||||
# AWS
|
||||
AWS_ACCESS_KEY_ID="op://Private/Hetzner/S3 Access Key ID"
|
||||
AWS_SECRET_ACCESS_KEY="op://Private/Hetzner/S3 Secret Access Key"
|
||||
S3_ENDPOINT="op://Private/Hetzner/S3 Endpoint"
|
||||
|
||||
# Gitea
|
||||
GITEA_TOKEN="op://Private/Gitea/token"
|
||||
@@ -25,4 +26,4 @@ DOJO_TOKEN="op://Private/DefectDojo/token"
|
||||
# Discourse
|
||||
DISCOURSE_URL="op://Environment Variables - Development/Ephemere/Discourse URL"
|
||||
DISCOURSE_API_KEY="op://Environment Variables - Development/Ephemere/Discourse Key"
|
||||
DISCOURSE_API_USERNAME="op://Environment Variables - Development/Ephemere/Discourse Username"
|
||||
DISCOURSE_API_USERNAME="op://Environment Variables - Development/Ephemere/Discourse Username"
|
||||
|
||||
@@ -17,6 +17,9 @@ build:
|
||||
lint:
|
||||
pnpm exec eslint src --max-warnings 0
|
||||
|
||||
format:
|
||||
pnpm exec eslint src --fix
|
||||
|
||||
test:
|
||||
@echo "No tests configured yet"
|
||||
@exit 0
|
||||
|
||||
@@ -12,11 +12,16 @@ import { getMimeType } from "../utils/mimeType.js";
|
||||
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
const endpoint = process.env.S3_ENDPOINT;
|
||||
|
||||
if (accessKeyId === undefined || secretAccessKey === undefined) {
|
||||
throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
|
||||
}
|
||||
|
||||
if (endpoint === undefined) {
|
||||
throw new Error("S3_ENDPOINT is not set");
|
||||
}
|
||||
|
||||
const dataDirectory = join(import.meta.dirname, "..", "..", "data");
|
||||
|
||||
/**
|
||||
@@ -172,7 +177,7 @@ if (!shouldProceed) {
|
||||
|
||||
const s3 = new S3Client({
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
endpoint: "https://hel1.your-objectstorage.com",
|
||||
endpoint: endpoint,
|
||||
region: "hel1",
|
||||
});
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import { getMimeType } from "../utils/mimeType.js";
|
||||
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
const endpoint = process.env.S3_ENDPOINT;
|
||||
|
||||
if (accessKeyId === undefined || secretAccessKey === undefined) {
|
||||
throw new Error(
|
||||
@@ -22,9 +23,13 @@ if (accessKeyId === undefined || secretAccessKey === undefined) {
|
||||
);
|
||||
}
|
||||
|
||||
if (endpoint === undefined) {
|
||||
throw new Error("S3_ENDPOINT is not set");
|
||||
}
|
||||
|
||||
const s3 = new S3Client({
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
endpoint: "https://hel1.your-objectstorage.com",
|
||||
endpoint: endpoint,
|
||||
region: "hel1",
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* @copyright NHCarrigan
|
||||
* @license Naomi's Public License
|
||||
* @author Naomi Carrigan
|
||||
*/
|
||||
import {
|
||||
S3Client, ListObjectsV2Command, DeleteObjectsCommand,
|
||||
type ListObjectsV2CommandOutput,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { input, confirm } from "@inquirer/prompts";
|
||||
import { SingleBar, Presets } from "cli-progress";
|
||||
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
const endpoint = process.env.S3_ENDPOINT;
|
||||
|
||||
if (accessKeyId === undefined || secretAccessKey === undefined) {
|
||||
throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
|
||||
}
|
||||
|
||||
if (endpoint === undefined) {
|
||||
throw new Error("S3_ENDPOINT is not set");
|
||||
}
|
||||
|
||||
// Get bucket name
|
||||
const bucketName = await input({
|
||||
message: "Enter the S3 bucket name:",
|
||||
validate: (value) => {
|
||||
if (value.trim() === "") {
|
||||
return "Bucket name cannot be empty";
|
||||
}
|
||||
// Basic S3 bucket name validation
|
||||
if (!/^[\d.a-z-]+$/.test(value)) {
|
||||
return `Bucket name can only contain lowercase letters, numbers, dots, and hyphens`;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
// Create S3 client
|
||||
const s3 = new S3Client({
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
endpoint: endpoint,
|
||||
region: "hel1",
|
||||
});
|
||||
|
||||
// First, count total objects to delete
|
||||
console.log("\nCounting objects in bucket...");
|
||||
let totalObjects = 0;
|
||||
let continuationToken: string | null = null;
|
||||
|
||||
do {
|
||||
const listCommand = new ListObjectsV2Command({
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
Bucket: bucketName,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
ContinuationToken: continuationToken ?? undefined,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
MaxKeys: 1000,
|
||||
});
|
||||
|
||||
const listResponse: ListObjectsV2CommandOutput = await s3.send(listCommand);
|
||||
totalObjects = totalObjects + (listResponse.Contents?.length ?? 0);
|
||||
continuationToken = listResponse.NextContinuationToken ?? null;
|
||||
} while (continuationToken !== null);
|
||||
|
||||
if (totalObjects === 0) {
|
||||
console.log("✨ No files found in the bucket!");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log(`\nFound ${totalObjects.toString()} object(s) to delete.`);
|
||||
|
||||
// Safety confirmation
|
||||
console.log(`\n⚠️ WARNING: This will DELETE ALL ${totalObjects.toString()} FILES in the bucket "${bucketName}"`);
|
||||
console.log("This action cannot be undone!\n");
|
||||
|
||||
// First confirmation - type bucket name
|
||||
await input({
|
||||
message: `Type the bucket name "${bucketName}" to confirm deletion:`,
|
||||
validate: (value) => {
|
||||
if (value !== bucketName) {
|
||||
return `Please type "${bucketName}" exactly to confirm`;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
// Second confirmation - yes/no
|
||||
const finalConfirm = await confirm({
|
||||
default: false,
|
||||
message: "Are you ABSOLUTELY sure you want to delete all files?",
|
||||
});
|
||||
|
||||
if (!finalConfirm) {
|
||||
console.log("❌ Operation cancelled. No files were deleted.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log("\n🚀 Starting deletion process...\n");
|
||||
|
||||
// Initialize progress bar
|
||||
const bar = new SingleBar({}, Presets.shades_classic);
|
||||
bar.start(totalObjects, 0);
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
continuationToken = null;
|
||||
|
||||
do {
|
||||
// List objects in the bucket
|
||||
const listCommand = new ListObjectsV2Command({
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
Bucket: bucketName,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
ContinuationToken: continuationToken ?? undefined,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
MaxKeys: 1000,
|
||||
});
|
||||
|
||||
const listResponse: ListObjectsV2CommandOutput = await s3.send(listCommand);
|
||||
|
||||
if (!listResponse.Contents || listResponse.Contents.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Prepare objects for deletion
|
||||
const objectsToDelete = listResponse.Contents.
|
||||
filter((object) => {
|
||||
return object.Key !== undefined;
|
||||
}).
|
||||
map((object) => {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
return { Key: object.Key };
|
||||
});
|
||||
|
||||
// Delete objects in batch
|
||||
const deleteCommand = new DeleteObjectsCommand({
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
Bucket: bucketName,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
Delete: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
Objects: objectsToDelete,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention -- AWS SDK
|
||||
Quiet: false,
|
||||
},
|
||||
});
|
||||
|
||||
const deleteResponse = await s3.send(deleteCommand);
|
||||
|
||||
const deletedCount = deleteResponse.Deleted?.length ?? 0;
|
||||
successCount = successCount + deletedCount;
|
||||
|
||||
// Check for errors
|
||||
if (deleteResponse.Errors && deleteResponse.Errors.length > 0) {
|
||||
errorCount = errorCount + deleteResponse.Errors.length;
|
||||
for (const error of deleteResponse.Errors) {
|
||||
console.error(`\n⚠️ Error deleting ${error.Key ?? ""}: ${error.Message ?? ""}`);
|
||||
}
|
||||
}
|
||||
|
||||
bar.increment(deletedCount);
|
||||
|
||||
continuationToken = listResponse.NextContinuationToken ?? null;
|
||||
} while (continuationToken !== null);
|
||||
|
||||
bar.stop();
|
||||
|
||||
console.log(
|
||||
`\n✅ Delete complete! ${successCount.toString()} succeeded, ${errorCount.toString()} failed.`,
|
||||
);
|
||||
@@ -11,11 +11,16 @@ import { getMimeType } from "../utils/mimeType.js";
|
||||
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
const endpoint = process.env.S3_ENDPOINT;
|
||||
|
||||
if (accessKeyId === undefined || secretAccessKey === undefined) {
|
||||
throw new Error("AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set");
|
||||
}
|
||||
|
||||
if (endpoint === undefined) {
|
||||
throw new Error("S3_ENDPOINT is not set");
|
||||
}
|
||||
|
||||
const fileName = await input({
|
||||
message:
|
||||
// eslint-disable-next-line stylistic/max-len -- Big boi string.
|
||||
@@ -40,7 +45,7 @@ if (uploadPath === "") {
|
||||
|
||||
const s3 = new S3Client({
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
endpoint: "https://hel1.your-objectstorage.com",
|
||||
endpoint: endpoint,
|
||||
region: "hel1",
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user