Unlink scripts submodule and commit contents as normal directory

scripts/services/bulk.js

@@ -0,0 +1,187 @@
+const FormData = require('form-data');
+const { parse } = require('csv-parse/sync');
+const { stringify } = require('csv-stringify/sync');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const { axiosBulk } = require('../services/axios.js');
+const { saveReportedIPs, markIPAsReported } = require('../services/cache.js');
+const ABUSE_STATE = require('./state.js');
+const { queueWrite, atomicWriteFile } = require('./writeQueue.js');
+const logger = require('../logger.js');
+
+const BULK_REPORT_BUFFER = new Map();
+const BUFFER_FILE = path.join(__dirname, '..', '..', 'tmp', 'bulk-report-buffer.csv');
+const OFFLINE_MODE = false;
+
+const ensureDirectoryExists = async filePath => {
+	try {
+		await fs.mkdir(path.dirname(filePath), { recursive: true });
+	} catch (err) {
+		if (err.code !== 'EEXIST') throw err;
+	}
+};
+
+const saveBufferToFile = () => queueWrite(BUFFER_FILE, async () => {
+	if (!BULK_REPORT_BUFFER.size) return;
+
+	const records = [];
+	for (const [ip, entry] of BULK_REPORT_BUFFER.entries()) {
+		records.push([ip, entry.categories, new Date(entry.timestamp).toISOString(), entry.comment]);
+	}
+
+	try {
+		const output = stringify(records, { header: true, columns: ['IP', 'Categories', 'ReportDate', 'Comment'], quoted: true });
+		await ensureDirectoryExists(BUFFER_FILE);
+		await atomicWriteFile(BUFFER_FILE, output);
+	} catch (err) {
+		logger.error(`Failed to write buffer file: ${err.message}`, { ping: true });
+	}
+});
+
+const loadBufferFromFile = async () => {
+	try {
+		await fs.access(BUFFER_FILE);
+
+		const fileContent = await fs.readFile(BUFFER_FILE, 'utf-8');
+		const records = parse(fileContent, { columns: false, from_line: 2, skip_empty_lines: true, trim: true });
+
+		for (const record of records) {
+			const [ip, categories, timestamp, comment] = record;
+			if (!ip || !timestamp) continue;
+			BULK_REPORT_BUFFER.set(ip, { categories, timestamp: new Date(timestamp).getTime(), comment });
+		}
+	} catch (err) {
+		if (err.code !== 'ENOENT') logger.error(`Failed to parse/load buffer file: ${err.message}`);
+	}
+};
+
+const sendBulkReport = async () => {
+	if (!BULK_REPORT_BUFFER.size) return;
+
+	logger.info(`Starting bulk report upload (${BULK_REPORT_BUFFER.size} IPs)...`);
+
+	const records = Array.from(BULK_REPORT_BUFFER.entries(), ([ip, entry]) => [
+		ip,
+		entry.categories,
+		new Date(entry.timestamp ?? Date.now()).toISOString(),
+		entry.comment ? (entry.comment.length > 1024 ? entry.comment.substring(0, 1021) + '...' : entry.comment) : '',
+	]);
+
+	const sendChunk = async (chunk, index = 0, total = 1) => {
+		const payload = stringify(chunk, {
+			header: true,
+			columns: ['IP', 'Categories', 'ReportDate', 'Comment'],
+			quoted: true,
+		});
+
+		const lines = payload.split('\n').length;
+		logger.success(`${total > 1 ? `Chunk ${index + 1}/${total}: ` : ''}Generated CSV with ${chunk.length} records, ${lines} lines`);
+
+		if (OFFLINE_MODE) {
+			if (lines > 10000) {
+				const error = new Error('Line limit exceeded');
+				error.response = {
+					status: 422,
+					data: { errors: [{ detail: `File uploaded has ${lines} lines which exceeds the line limit of 10,000 lines.`, status: 422 }] },
+				};
+				throw error;
+			}
+
+			const saved = chunk.length;
+			logger.success(`${total > 1 ? `Chunk ${index + 1}/${total}: ` : 'Sent bulk report: '}${saved} accepted, 0 rejected [OFFLINE]`, { discord: true });
+			return;
+		}
+
+		if (lines > 10000) {
+			const error = new Error('Line limit exceeded');
+			error.response = {
+				status: 422,
+				data: { errors: [{ detail: `File uploaded has ${lines} lines which exceeds the line limit of 10,000 lines.`, status: 422 }] },
+			};
+			throw error;
+		}
+
+		const form = new FormData();
+		form.append('csv', Buffer.from(payload), {
+			filename: total > 1 ? `report-chunk-${index + 1}.csv` : 'report.csv',
+			contentType: 'text/csv',
+		});
+
+		const { data } = await axiosBulk.post('/bulk-report', form, { headers: { ...form.getHeaders() } });
+		const saved = data?.data?.savedReports ?? 0;
+		const failed = data?.data?.invalidReports?.length ?? 0;
+
+		logger.success(`${total > 1 ? `Chunk ${index + 1}/${total}: ` : 'Sent bulk report: '}${saved} accepted, ${failed} rejected`, { discord: true });
+
+		if (failed > 0) {
+			const prefix = total > 1 ? `Chunk ${index + 1}: ` : '';
+			data.data.invalidReports.forEach(fail => {
+				logger.warn(`${prefix}Rejected [Row ${fail.rowNumber}] ${fail.input} -> ${fail.error}`);
+			});
+		}
+	};
+
+	const isLineLimitError = err =>
+		err.response?.status === 422 &&
+		err.response?.data?.errors?.some(e => e.detail.includes('exceeds the line limit'));
+
+	let success = false;
+
+	try {
+		await sendChunk(records);
+		success = true;
+	} catch (err) {
+		if (isLineLimitError(err)) {
+			const chunks = [];
+			const estimatedLinesPerRecord = 10;
+			const chunkSize = Math.max(1, Math.floor(9000 / estimatedLinesPerRecord));
+
+			for (let i = 0; i < records.length; i += chunkSize) {
+				chunks.push(records.slice(i, i + chunkSize));
+			}
+
+			logger.info(`File too large. Splitting ${records.length} records into ${chunks.length} chunks (~${chunkSize} records each)`, { discord: true });
+
+			let allChunksSuccessful = true;
+			for (let i = 0; i < chunks.length; i++) {
+				try {
+					await sendChunk(chunks[i], i, chunks.length);
+				} catch (chunkErr) {
+					logger.error(`Chunk ${i + 1} failed: ${chunkErr.response?.data?.errors?.[0]?.detail || chunkErr.message}`);
+					allChunksSuccessful = false;
+				}
+			}
+			success = allChunksSuccessful;
+		} else {
+			const errorMsg = err.response?.data?.errors
+				? err.response.data.errors.map(e => e.detail).join(', ')
+				: err.response?.data ? JSON.stringify(err.response.data) : err.stack;
+
+			logger.error(errorMsg);
+		}
+	}
+
+	if (success) {
+		for (const ip of BULK_REPORT_BUFFER.keys()) markIPAsReported(ip);
+		await saveReportedIPs();
+		BULK_REPORT_BUFFER.clear();
+
+		try {
+			await fs.unlink(BUFFER_FILE);
+		} catch (err) {
+			// Ignore file deletion errors - file may not exist or already deleted
+			if (err.code !== 'ENOENT') logger.warn(`Failed to delete buffer file: ${err.message}`);
+		}
+
+		ABUSE_STATE.sentBulk = true;
+	} else {
+		logger.error('Bulk upload failed, keeping buffer file for retry', { ping: true });
+	}
+};
+
+module.exports = Object.freeze({
+	saveBufferToFile,
+	loadBufferFromFile,
+	sendBulkReport,
+	BULK_REPORT_BUFFER,
+});