Improve output of processing script

reference/process_issues.mjs

@@ -10,8 +10,8 @@
  */
 
 import { JSDOM } from 'jsdom';
-import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
-import { join, basename } from 'node:path';
+import { join, basename, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
@@ -29,6 +29,7 @@ const { buildIssueMarkdownDocument } = await import(
 );
 
 const OUTPUT_ROOT = join(REPO_ROOT, 'web', 'data', 'issues');
+const MODIFIED_FILE_LABEL = 'Modified file:';
 
 function readHtmlFiles(dirPath) {
     try {
@@ -38,16 +39,8 @@ function readHtmlFiles(dirPath) {
     }
 }
 
-function getProductNames() {
+function getInputEntries(referenceRoot, productName) {
-    return readdirSync(__dirname, { withFileTypes: true })
+    const productRoot = join(referenceRoot, productName);
-        .filter(entry => entry.isDirectory())
-        .map(entry => entry.name)
-        .filter(name => name !== '.' && name !== '..')
-        .sort();
-}
-
-function getInputEntries(productName) {
-    const productRoot = join(__dirname, productName);
     const entries = [];
     for (const issueType of ['addressed', 'known']) {
         const issueTypeDir = join(productRoot, issueType);
@@ -64,18 +57,60 @@ function getInputEntries(productName) {
     return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
 }
 
-let writtenCount = 0;
+function createNoChangeReporter(stdout) {
-let skippedNoIssues = 0;
+    let noChangeCount = 0;
-let processedProducts = 0;
+    let isShowingCounter = false;
 
-const allEntries = getProductNames().flatMap(product =>
+    return {
-    getInputEntries(product).map(entry => ({ ...entry, product }))
+        increment() {
-);
+            noChangeCount++;
-const totalEntries = allEntries.length;
+            const noun = noChangeCount === 1 ? 'file' : 'files';
-let processedEntries = 0;
+            stdout.write(`\rNo changes for: ${noChangeCount} ${noun}`);
-const seenProducts = new Set();
+            isShowingCounter = true;
+        },
+        flush() {
+            if (!isShowingCounter) {
+                return;
+            }
+            stdout.write('\n');
+            isShowingCounter = false;
+        },
+        getCount() {
+            return noChangeCount;
+        }
+    };
+}
 
-for (const entry of allEntries) {
+export function formatFileStatusLine(status, fileLabel) {
+    if (status === 'modified') {
+        return `${MODIFIED_FILE_LABEL} ${fileLabel}`;
+    }
+    if (status === 'new') {
+        return `${'New file:'.padEnd(MODIFIED_FILE_LABEL.length)} ${fileLabel}`;
+    }
+    throw new Error(`Unsupported file status: ${status}`);
+}
+
+export function processIssues({
+    referenceRoot = __dirname,
+    outputRoot = OUTPUT_ROOT,
+    stdout = process.stdout,
+    logger = console,
+} = {}) {
+    let newCount = 0;
+    let modifiedCount = 0;
+    let skippedNoIssues = 0;
+    let processedProducts = 0;
+
+    const allEntries = getProductNames(referenceRoot).flatMap(product =>
+        getInputEntries(referenceRoot, product).map(entry => ({ ...entry, product }))
+    );
+    const totalEntries = allEntries.length;
+    let processedEntries = 0;
+    const seenProducts = new Set();
+    const noChangeReporter = createNoChangeReporter(stdout);
+
+    for (const entry of allEntries) {
         const product = entry.product;
         if (!seenProducts.has(product)) {
             seenProducts.add(product);
@@ -90,7 +125,8 @@ for (const entry of allEntries) {
         const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType });
         processedEntries++;
         if (parsedIssues.length === 0) {
-        console.warn(`[${processedEntries}/${totalEntries}] No issues parsed from ${basename(filePath)} — skipping`);
+            noChangeReporter.flush();
+            logger.warn(`[${processedEntries}/${totalEntries}] No issues parsed from ${basename(filePath)} — skipping`);
             skippedNoIssues++;
             continue;
         }
@@ -102,24 +138,69 @@ for (const entry of allEntries) {
             issues: parsedIssues,
         });
 
-    const outDir = join(OUTPUT_ROOT, product, issueType);
+        const outDir = join(outputRoot, product, issueType);
         mkdirSync(outDir, { recursive: true });
 
         const outFile = join(outDir, `${version}.md`);
+        if (existsSync(outFile) && readFileSync(outFile, 'utf-8') === markdown) {
+            noChangeReporter.increment();
+            continue;
+        }
+
+        noChangeReporter.flush();
+        const fileLabel = `${product}-${version}`;
+        const isNewFile = !existsSync(outFile);
         writeFileSync(outFile, markdown, 'utf-8');
-    writtenCount++;
+        if (isNewFile) {
-    console.log(`[${processedEntries}/${totalEntries}] ${product} ${version} (${issueType})`);
+            newCount++;
+            logger.log(formatFileStatusLine('new', fileLabel));
+        } else {
+            modifiedCount++;
+            logger.log(formatFileStatusLine('modified', fileLabel));
+        }
+    }
+
+    if (processedProducts === 0) {
+        noChangeReporter.flush();
+        logger.error(`No HTML files found under ${referenceRoot}.`);
+        return {
+            processedProducts,
+            newCount,
+            modifiedCount,
+            noChangeCount: noChangeReporter.getCount(),
+            skippedNoIssues,
+        };
+    }
+
+    const writtenCount = newCount + modifiedCount;
+    noChangeReporter.flush();
+    logger.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`);
+    if (skippedNoIssues > 0) {
+        logger.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`);
+    }
+    if (writtenCount > 0) {
+        logger.log('Next step: python scripts/update_products_from_issues.py');
+    }
+
+    return {
+        processedProducts,
+        newCount,
+        modifiedCount,
+        noChangeCount: noChangeReporter.getCount(),
+        skippedNoIssues,
+    };
 }
 
-if (processedProducts === 0) {
+function getProductNames(referenceRoot) {
-    console.error(`No HTML files found under ${__dirname}.`);
+    return readdirSync(referenceRoot, { withFileTypes: true })
-    process.exit(0);
+        .filter(entry => entry.isDirectory())
+        .map(entry => entry.name)
+        .filter(name => name !== '.' && name !== '..')
+        .sort();
 }
 
-console.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`);
+const isMainModule = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
-if (skippedNoIssues > 0) {
+
-    console.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`);
+if (isMainModule) {
-}
+    processIssues();
-if (writtenCount > 0) {
-    console.log('Next step: python scripts/update_products_from_issues.py');
 }

test/process-parser.test.mjs

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
-import { readFileSync, readdirSync } from 'node:fs';
+import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { JSDOM } from 'jsdom';
@@ -10,6 +11,7 @@ globalThis.DOMParser = window.DOMParser;
 
 const { parseIssuesFromHtmlTable } = await import('../web/js/process.js');
 const { buildIssueMarkdownDocument } = await import('../web/js/markdown.js');
+const { formatFileStatusLine, processIssues } = await import('../reference/process_issues.mjs');
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
 const fixturesDir = join(__dirname, 'fixtures');
@@ -69,3 +71,82 @@ fixtures.forEach(fixtureId => {
         assert.equal(markdown.trim(), expectedMarkdown.trim());
     });
 });
+
+test('processIssues reports unchanged, modified, and new files distinctly', () => {
+    const tempRoot = mkdtempSync(join(tmpdir(), 'firewallissues-process-'));
+    const referenceRoot = join(tempRoot, 'reference');
+    const outputRoot = join(tempRoot, 'web', 'data', 'issues');
+    const fixtureRoot = join(fixturesDir, 'PAN-OS-10.2.1-addressed');
+    const modifiedFixtureRoot = join(fixturesDir, 'PAN-OS-10.1.11-known');
+    const newFixtureRoot = join(fixturesDir, 'PAN-OS-12.1.2-addressed');
+    const stdoutChunks = [];
+    const logs = [];
+
+    try {
+        mkdirSync(join(referenceRoot, 'PAN-OS', 'addressed'), { recursive: true });
+        mkdirSync(join(referenceRoot, 'PAN-OS', 'known'), { recursive: true });
+
+        writeFileSync(
+            join(referenceRoot, 'PAN-OS', 'addressed', '10.2.1.html'),
+            readFileSync(join(fixtureRoot, 'input.html'), 'utf-8')
+        );
+        writeFileSync(
+            join(referenceRoot, 'PAN-OS', 'known', '10.1.11.html'),
+            readFileSync(join(modifiedFixtureRoot, 'input.html'), 'utf-8')
+        );
+        writeFileSync(
+            join(referenceRoot, 'PAN-OS', 'addressed', '12.1.2.html'),
+            readFileSync(join(newFixtureRoot, 'input.html'), 'utf-8')
+        );
+
+        mkdirSync(join(outputRoot, 'PAN-OS', 'addressed'), { recursive: true });
+        mkdirSync(join(outputRoot, 'PAN-OS', 'known'), { recursive: true });
+        writeFileSync(
+            join(outputRoot, 'PAN-OS', 'addressed', '10.2.1.md'),
+            readFileSync(join(fixtureRoot, 'expected.md'), 'utf-8')
+        );
+        writeFileSync(join(outputRoot, 'PAN-OS', 'known', '10.1.11.md'), 'old content');
+
+        const summary = processIssues({
+            referenceRoot,
+            outputRoot,
+            stdout: {
+                write(chunk) {
+                    stdoutChunks.push(chunk);
+                }
+            },
+            logger: {
+                log(message) {
+                    logs.push(message);
+                },
+                warn(message) {
+                    logs.push(message);
+                },
+                error(message) {
+                    logs.push(message);
+                }
+            }
+        });
+
+        assert.deepEqual(summary, {
+            processedProducts: 1,
+            newCount: 1,
+            modifiedCount: 1,
+            noChangeCount: 1,
+            skippedNoIssues: 0,
+        });
+        assert.equal(stdoutChunks.join(''), '\rNo changes for: 1 file\n');
+        assert.ok(logs.includes(formatFileStatusLine('modified', 'PAN-OS-10.1.11')));
+        assert.ok(logs.includes(formatFileStatusLine('new', 'PAN-OS-12.1.2')));
+        assert.equal(
+            readFileSync(join(outputRoot, 'PAN-OS', 'known', '10.1.11.md'), 'utf-8').trim(),
+            readFileSync(join(modifiedFixtureRoot, 'expected.md'), 'utf-8').trim()
+        );
+        assert.equal(
+            readFileSync(join(outputRoot, 'PAN-OS', 'addressed', '12.1.2.md'), 'utf-8').trim(),
+            readFileSync(join(newFixtureRoot, 'expected.md'), 'utf-8').trim()
+        );
+    } finally {
+        rmSync(tempRoot, { recursive: true, force: true });
+    }
+});