From 38e7f15d93bf768f8fb8b9bb36153a711dad41a0 Mon Sep 17 00:00:00 2001 From: Naomi Carrigan Date: Tue, 13 Jan 2026 19:07:03 -0800 Subject: [PATCH] feat: add scripts for managing discourse --- prod.env | 5 + src/discourse/bulkUpdateCategories.ts | 127 ++++++++++++++ src/discourse/closeOldTopics.ts | 240 ++++++++++++++++++++++++++ src/interfaces/discourse.ts | 188 ++++++++++++++++++++ src/utils/backoffAndRetry.ts | 10 +- 5 files changed, 567 insertions(+), 3 deletions(-) create mode 100644 src/discourse/bulkUpdateCategories.ts create mode 100644 src/discourse/closeOldTopics.ts create mode 100644 src/interfaces/discourse.ts diff --git a/prod.env b/prod.env index 44d5544..8ad11cb 100644 --- a/prod.env +++ b/prod.env @@ -20,3 +20,8 @@ GITEA_TOKEN="op://Private/Gitea/token" # DefectDojo DOJO_TOKEN="op://Private/DefectDojo/token" + +# Discourse +DISCOURSE_URL="op://Environment Variables - Development/Ephemere/Discourse URL" +DISCOURSE_API_KEY="op://Environment Variables - Development/Ephemere/Discourse Key" +DISCOURSE_API_USERNAME="op://Environment Variables - Development/Ephemere/Discourse Username" \ No newline at end of file diff --git a/src/discourse/bulkUpdateCategories.ts b/src/discourse/bulkUpdateCategories.ts new file mode 100644 index 0000000..0110298 --- /dev/null +++ b/src/discourse/bulkUpdateCategories.ts @@ -0,0 +1,127 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +import { backoffAndRetry } from "../utils/backoffAndRetry.js"; +import type { CategoryGet, CategoryList } from "../interfaces/discourse.js"; + +const discourseUrl = process.env.DISCOURSE_URL; +const discourseApiKey = process.env.DISCOURSE_API_KEY; +const discourseApiUsername = process.env.DISCOURSE_API_USERNAME; + +if ( + discourseUrl === undefined + || discourseApiKey === undefined + || discourseApiUsername === undefined +) { + throw new Error( + "DISCOURSE_URL, DISCOURSE_API_KEY, or DISCOURSE_API_USERNAME is not set", + ); +} + +const categoryResponse = await backoffAndRetry( + `${discourseUrl}/categories.json`, + { + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Key": discourseApiKey, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Username": discourseApiUsername, + }, + }, +); + +if (categoryResponse === null) { + throw new Error("Failed to get categories"); +} +const categories = categoryResponse; + +// Collect all categories including subcategories +const allCategories: Array<{ id: number; name: string }> = []; +const processedIds = new Set(); + +// Process top-level categories and their subcategories +for (const category of categories.category_list.categories) { + // Add top-level category if not already processed + if (!processedIds.has(category.id)) { + allCategories.push({ id: category.id, name: category.name }); + processedIds.add(category.id); + } + + if (!Array.isArray(category.subcategory_ids)) { + continue; + } + + // Add subcategories if they exist + for (const subcategoryId of category.subcategory_ids) { + if (!processedIds.has(subcategoryId)) { + const subcategoryRequest = await backoffAndRetry( + `${discourseUrl}/c/${subcategoryId.toString()}/show.json`, + { + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Key": discourseApiKey, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Username": discourseApiUsername, + }, + }, + ); + if (subcategoryRequest === null) { + console.error(`Failed to get subcategory ${subcategoryId.toString()}`); + continue; + } + if (subcategoryRequest.category?.name === undefined) { + console.error(`Failed to get subcategory ${subcategoryId.toString()}`); + console.log(Object.keys(subcategoryRequest)); + continue; + } + allCategories.push({ + id: subcategoryId, + name: subcategoryRequest.category.name, + }); + processedIds.add(subcategoryId); + } + } +} + +// Update all categories (including subcategories) +for (const category of allCategories) { + const { id, name } = category; + console.log(`Updating category ${id.toString()}: ${name}`); + const updateRequest = await backoffAndRetry( + `${discourseUrl}/categories/${id.toString()}`, + { + body: JSON.stringify({ + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required body format. + auto_close_based_on_last_post: true, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required body format. + auto_close_hours: 672, + name: name, + }), + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Key": discourseApiKey, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Username": discourseApiUsername, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Content-Type": "application/json", + }, + method: "PUT", + }, + ); + if (updateRequest === null) { + console.error(`Failed to update category ${id.toString()}`); + continue; + } + if (updateRequest.category?.auto_close_hours === undefined) { + console.error(`Failed to update category ${id.toString()}`); + console.error(JSON.stringify(updateRequest, null, 2)); + continue; + } + const { auto_close_hours: returnedAutoCloseHours } = updateRequest.category; + console.log( + `Updated category ${id.toString()}: ${name} to auto_close after ${returnedAutoCloseHours.toString()} hours`, + ); +} diff --git a/src/discourse/closeOldTopics.ts b/src/discourse/closeOldTopics.ts new file mode 100644 index 0000000..daea1a1 --- /dev/null +++ b/src/discourse/closeOldTopics.ts @@ -0,0 +1,240 @@ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +import { backoffAndRetry } from "../utils/backoffAndRetry.js"; +import { sleep } from "../utils/sleep.js"; +import type { Topic, TopicList } from "../interfaces/discourse.js"; + +const discourseUrl = process.env.DISCOURSE_URL; +const discourseApiKey = process.env.DISCOURSE_API_KEY; +const discourseApiUsername = process.env.DISCOURSE_API_USERNAME; + +if ( + discourseUrl === undefined + || discourseApiKey === undefined + || discourseApiUsername === undefined +) { + throw new Error( + "DISCOURSE_URL, DISCOURSE_API_KEY, or DISCOURSE_API_USERNAME is not set", + ); +} + +const apiHeaders = { + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Key": discourseApiKey, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Api-Username": discourseApiUsername, +}; + +/** + * Fetches a single page of topics from Discourse. + * @param page - The page number to fetch. + * @returns The topics from that page, or null if the request failed. + */ +const fetchTopicsPage = async(page: number): Promise | null> => { + const url = `${discourseUrl}/latest.json?page=${page.toString()}`; + console.log(`Fetching topics page ${page.toString()}...`); + + const response = await backoffAndRetry( + url, + { + headers: apiHeaders, + }, + ); + + if (response === null) { + console.error(`Failed to fetch page ${page.toString()}`); + return null; + } + + const { topic_list: topicList } = response; + const { topics } = topicList; + console.log( + `Page ${page.toString()}: Received ${topics.length.toString()} topics`, + ); + + return topics; +}; + +/** + * Gets the last activity date for a topic. + * @param topic - The topic to get the activity date for. + * @returns The date when the topic was last active. + */ +const getLastActivityDate = (topic: Topic): Date => { + if (topic.last_posted_at !== null) { + return new Date(topic.last_posted_at); + } + + if (topic.bumped_at !== null) { + return new Date(topic.bumped_at); + } + + return new Date(topic.created_at); +}; + +/** + * Fetches open topics from Discourse until we hit topics older than 6 months. + * The /latest.json endpoint returns topics in a nested structure, so we need + * to handle it differently than standard paginated endpoints. + * @param sixMonthsAgo - The date cutoff for stopping pagination (6 months ago). + * @returns Open topics that are newer than 6 months old. + */ +const fetchAllOpenTopics = async(sixMonthsAgo: Date): Promise> => { + const allTopics: Array = []; + let page = 0; + let topics: Array | null = await fetchTopicsPage(page); + + while (topics !== null && topics.length > 0) { + let foundOldTopic = false; + + for (const topic of topics) { + // Skip closed topics + if (topic.closed) { + continue; + } + + const lastActivityDate = getLastActivityDate(topic); + + // If we hit a topic older than 6 months, stop fetching + if (lastActivityDate < sixMonthsAgo) { + foundOldTopic = true; + console.log( + `Found topic older than 6 months (${lastActivityDate.toISOString()}), stopping pagination.`, + ); + break; + } + + allTopics.push(topic); + } + + if (foundOldTopic) { + break; + } + + // Continue to next page + page = page + 1; + topics = await fetchTopicsPage(page); + } + + console.log(`Total open topics fetched: ${allTopics.length.toString()}`); + return allTopics; +}; + +/** + * Closes a topic by ID. + * @param topicId - The ID of the topic to close. + * @param topicTitle - The title of the topic (for logging). + * @returns Whether the topic was successfully closed. + */ +const closeTopic = async( + topicId: number, + topicTitle: string, +): Promise => { + const url = `${discourseUrl}/t/${topicId.toString()}/status`; + console.log(`Closing topic ${topicId.toString()}: ${topicTitle}`); + + const response = await fetch(url, { + body: JSON.stringify({ + enabled: true, + status: "closed", + }), + headers: { + ...apiHeaders, + // eslint-disable-next-line @typescript-eslint/naming-convention -- Required header format. + "Content-Type": "application/json", + }, + method: "PUT", + }); + + if (!response.ok) { + if (response.status === 429) { + console.log("Rate limited, waiting 5 seconds..."); + await sleep(5000); + return await closeTopic(topicId, topicTitle); + } + console.error( + `Failed to close topic ${topicId.toString()}: ${response.status.toString()} ${response.statusText}`, + ); + return false; + } + + console.log(`Successfully closed topic ${topicId.toString()}: ${topicTitle}`); + return true; +}; + +// Main execution +const daysInactive = 28; + +/** + * Approximately 6 months in days. + */ +const daysSixMonths = 180; + +const cutoffDate = new Date(); +cutoffDate.setDate(cutoffDate.getDate() - daysInactive); + +const sixMonthsAgo = new Date(); +sixMonthsAgo.setDate(sixMonthsAgo.getDate() - daysSixMonths); + +console.log(`Fetching open topics (stopping at topics older than 6 months)...`); +console.log( + `Will close topics inactive for ${daysInactive.toString()}+ days but less than ${daysSixMonths.toString()} days old`, +); +console.log( + `Cutoff date: ${cutoffDate.toISOString()} (topics inactive for ${daysInactive.toString()} days)`, +); +console.log( + `Stop fetching at: ${sixMonthsAgo.toISOString()} (6 months ago)`, +); + +const openTopics = await fetchAllOpenTopics(sixMonthsAgo); + +console.log(`\nChecking ${openTopics.length.toString()} open topics for inactivity...`); + +const topicsToClose: Array = []; + +for (const topic of openTopics) { + const lastActivityDate = getLastActivityDate(topic); + + /** + * Only close topics that are inactive for 28+ days but less than 6 months old. + */ + if (lastActivityDate < cutoffDate && lastActivityDate >= sixMonthsAgo) { + topicsToClose.push(topic); + } +} + +console.log(`Found ${topicsToClose.length.toString()} topics to close\n`); + +if (topicsToClose.length === 0) { + console.log("No topics need to be closed."); + process.exit(0); +} + +// Close topics with rate limiting +let closedCount = 0; +let failedCount = 0; + +for (const topic of topicsToClose) { + const success = await closeTopic(topic.id, topic.title); + if (success) { + closedCount = closedCount + 1; + } else { + failedCount = failedCount + 1; + } + + /** + * Add a small delay between requests to respect rate limits. + * (backoffAndRetry handles 429s, but we want to avoid hitting them). + */ + await sleep(500); +} + +console.log(`\nCompleted:`); +console.log(` Closed: ${closedCount.toString()}`); +console.log(` Failed: ${failedCount.toString()}`); +console.log(` Total: ${topicsToClose.length.toString()}`); diff --git a/src/interfaces/discourse.ts b/src/interfaces/discourse.ts new file mode 100644 index 0000000..6bc1182 --- /dev/null +++ b/src/interfaces/discourse.ts @@ -0,0 +1,188 @@ +/* eslint-disable @typescript-eslint/naming-convention -- The names of the properties match the API responses. */ +/** + * @copyright NHCarrigan + * @license Naomi's Public License + * @author Naomi Carrigan + */ + +interface CategoryList { + category_list: { + can_create_category: boolean; + can_create_topic: boolean; + categories: Array<{ + id: number; + name: string; + color: string; + text_color: string; + style_type: string; + emoji: string; + icon: string; + slug: string; + topic_count: number; + post_count: number; + position: number; + description: string; + description_text: string; + description_excerpt: string; + topic_url: string; + read_restricted: boolean; + permission: number; + notification_level: number; + can_edit: boolean; + topic_template: string; + has_children: boolean; + subcategory_count: number; + sort_order: string; + sort_ascending: string; + show_subcategory_list: boolean; + num_featured_topics: number; + default_view: string; + subcategory_list_style: string; + default_top_period: string; + default_list_filter: string; + minimum_required_tags: number; + navigate_to_first_post_after_read: boolean; + topics_day: number; + topics_week: number; + topics_month: number; + topics_year: number; + topics_all_time: number; + is_uncategorized: boolean; + subcategory_ids: Array; + subcategory_list: Array<{ + id: number; + name: string; + }>; + uploaded_logo: string; + uploaded_logo_dark: string; + uploaded_background: string; + uploaded_background_dark: string; + }>; + }; +} + +interface CategoryGet { + category?: { + id: number; + name: string; + color: string; + text_color: string; + style_type: string; + emoji: string; + icon: string; + slug: string; + topic_count: number; + post_count: number; + position: number; + description: string; + description_text: string; + description_excerpt: string; + topic_url: string; + read_restricted: boolean; + permission: number; + notification_level: number; + can_edit: boolean; + topic_template: string; + form_template_ids: Array; + has_children: boolean; + subcategory_count: number; + sort_order: string; + sort_ascending: string; + show_subcategory_list: boolean; + num_featured_topics: number; + default_view: string; + subcategory_list_style: string; + default_top_period: string; + default_list_filter: string; + minimum_required_tags: number; + navigate_to_first_post_after_read: boolean; + custom_fields: Record; + allowed_tags: Array; + allowed_tag_groups: Array; + allow_global_tags: boolean; + required_tag_groups: Array<{ + name: string; + min_count: number; + }>; + category_setting: { + auto_bump_cooldown_days: number; + num_auto_bump_daily: number; + require_reply_approval: boolean; + require_topic_approval: boolean; + }; + category_localizations: Array; + read_only_banner: string; + available_groups: Array; + auto_close_hours: string; + auto_close_based_on_last_post: boolean; + allow_unlimited_owner_edits_on_first_post: boolean; + default_slow_mode_seconds: string; + group_permissions: Array<{ + permission_type: number; + group_name: string; + group_id: number; + }>; + email_in: string; + email_in_allow_strangers: boolean; + mailinglist_mirror: boolean; + all_topics_wiki: boolean; + can_delete: boolean; + allow_badges: boolean; + topic_featured_link_allowed: boolean; + search_priority: number; + uploaded_logo: string; + uploaded_logo_dark: string; + uploaded_background: string; + uploaded_background_dark: string; + }; +} + +interface Topic { + id: number; + title: string; + fancy_title: string; + slug: string; + posts_count: number; + reply_count: number; + highest_post_number: number; + image_url: string | null; + created_at: string; + last_posted_at: string | null; + bumped: boolean; + bumped_at: string | null; + archetype: string; + unseen: boolean; + pinned: boolean; + unpinned: boolean | null; + visible: boolean; + closed: boolean; + archived: boolean; + bookmarked: boolean | null; + liked: boolean | null; + tags: Array; + tags_descriptions: Record; + like_count: number; + views: number; + category_id: number; + featured_link: string | null; + has_accepted_answer: boolean; + posters: Array<{ + extras: string | null; + description: string; + user_id: number; + primary_group_id: number | null; + }>; +} + +interface TopicList { + topic_list: { + can_create_topic: boolean; + draft: string | null; + draft_key: string; + draft_sequence: number; + per_page: number; + topics: Array; + }; +} + +export type { CategoryList, CategoryGet, Topic, TopicList }; diff --git a/src/utils/backoffAndRetry.ts b/src/utils/backoffAndRetry.ts index fbec676..d511b14 100644 --- a/src/utils/backoffAndRetry.ts +++ b/src/utils/backoffAndRetry.ts @@ -9,23 +9,27 @@ import { sleep } from "./sleep.js"; /** * Wraps the native fetch method in logic to back off * and retry on 429 errors. + * @type {T} - The type of the response. * @param url - The URL to fetch. * @param options - The fetch options. + * @returns The response, or null on error. */ export const backoffAndRetry - = async(url: string, options: RequestInit = {}): Promise => { + = async(url: string, options: RequestInit = {}): Promise => { try { const response = await fetch(url, options); if (!response.ok) { if (response.status === 429) { await sleep(5000); - await backoffAndRetry(url, options); - return; + return await backoffAndRetry(url, options); } throw new Error(`Request failed with status ${response.status.toString()}`); } + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- This is a workaround to avoid type errors. + return await response.json() as T; } catch (error) { console.error(`Fetch error: ${JSON.stringify(error, null, 2)}`); + return null; } };