aaron.axvig/firewallissues
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve output of processing script

Browse Source
This commit is contained in:
aaron.axvig
2026-05-21 08:36:38 -05:00
parent cbea768c6e
commit 625583828b
2 changed files with 222 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reference/process_issues.mjs
+105 -24
View File
@@ -10,8 +10,8 @@
*/
import { JSDOM } from 'jsdom';
import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
import { join, basename } from 'node:path';
import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
import { join, basename, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
@@ -29,6 +29,7 @@ const { buildIssueMarkdownDocument } = await import(
);
const OUTPUT_ROOT = join(REPO_ROOT, 'web', 'data', 'issues');
const MODIFIED_FILE_LABEL = 'Modified file:';
function readHtmlFiles(dirPath) {
try {
@@ -38,16 +39,8 @@ function readHtmlFiles(dirPath) {
}
}
function getProductNames() {
return readdirSync(__dirname, { withFileTypes: true })
.filter(entry => entry.isDirectory())
.map(entry => entry.name)
.filter(name => name !== '.' && name !== '..')
.sort();
}
function getInputEntries(productName) {
const productRoot = join(__dirname, productName);
function getInputEntries(referenceRoot, productName) {
const productRoot = join(referenceRoot, productName);
const entries = [];
for (const issueType of ['addressed', 'known']) {
const issueTypeDir = join(productRoot, issueType);
@@ -64,16 +57,58 @@ function getInputEntries(productName) {
return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
}
let writtenCount = 0;
function createNoChangeReporter(stdout) {
let noChangeCount = 0;
let isShowingCounter = false;
return {
increment() {
noChangeCount++;
const noun = noChangeCount === 1 ? 'file' : 'files';
stdout.write(`\rNo changes for: ${noChangeCount} ${noun}`);
isShowingCounter = true;
},
flush() {
if (!isShowingCounter) {
return;
}
stdout.write('\n');
isShowingCounter = false;
},
getCount() {
return noChangeCount;
}
};
}
export function formatFileStatusLine(status, fileLabel) {
if (status === 'modified') {
return `${MODIFIED_FILE_LABEL} ${fileLabel}`;
}
if (status === 'new') {
return `${'New file:'.padEnd(MODIFIED_FILE_LABEL.length)} ${fileLabel}`;
}
throw new Error(`Unsupported file status: ${status}`);
}
export function processIssues({
referenceRoot = __dirname,
outputRoot = OUTPUT_ROOT,
stdout = process.stdout,
logger = console,
} = {}) {
let newCount = 0;
let modifiedCount = 0;
let skippedNoIssues = 0;
let processedProducts = 0;
const allEntries = getProductNames().flatMap(product =>
getInputEntries(product).map(entry => ({ ...entry, product }))
const allEntries = getProductNames(referenceRoot).flatMap(product =>
getInputEntries(referenceRoot, product).map(entry => ({ ...entry, product }))
);
const totalEntries = allEntries.length;
let processedEntries = 0;
const seenProducts = new Set();
const noChangeReporter = createNoChangeReporter(stdout);
for (const entry of allEntries) {
const product = entry.product;
@@ -90,7 +125,8 @@ for (const entry of allEntries) {
const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType });
processedEntries++;
if (parsedIssues.length === 0) {
console.warn(`[${processedEntries}/${totalEntries}] No issues parsed from ${basename(filePath)} — skipping`);
noChangeReporter.flush();
logger.warn(`[${processedEntries}/${totalEntries}] No issues parsed from ${basename(filePath)} — skipping`);
skippedNoIssues++;
continue;
}
@@ -102,24 +138,69 @@ for (const entry of allEntries) {
issues: parsedIssues,
});
const outDir = join(OUTPUT_ROOT, product, issueType);
const outDir = join(outputRoot, product, issueType);
mkdirSync(outDir, { recursive: true });
const outFile = join(outDir, `${version}.md`);
if (existsSync(outFile) && readFileSync(outFile, 'utf-8') === markdown) {
noChangeReporter.increment();
continue;
}
noChangeReporter.flush();
const fileLabel = `${product}-${version}`;
const isNewFile = !existsSync(outFile);
writeFileSync(outFile, markdown, 'utf-8');
writtenCount++;
console.log(`[${processedEntries}/${totalEntries}] ${product} ${version} (${issueType})`);
if (isNewFile) {
newCount++;
logger.log(formatFileStatusLine('new', fileLabel));
} else {
modifiedCount++;
logger.log(formatFileStatusLine('modified', fileLabel));
}
}
if (processedProducts === 0) {
console.error(`No HTML files found under ${__dirname}.`);
process.exit(0);
noChangeReporter.flush();
logger.error(`No HTML files found under ${referenceRoot}.`);
return {
processedProducts,
newCount,
modifiedCount,
noChangeCount: noChangeReporter.getCount(),
skippedNoIssues,
};
}
console.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`);
const writtenCount = newCount + modifiedCount;
noChangeReporter.flush();
logger.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`);
if (skippedNoIssues > 0) {
console.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`);
logger.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`);
}
if (writtenCount > 0) {
console.log('Next step: python scripts/update_products_from_issues.py');
logger.log('Next step: python scripts/update_products_from_issues.py');
}
return {
processedProducts,
newCount,
modifiedCount,
noChangeCount: noChangeReporter.getCount(),
skippedNoIssues,
};
}
function getProductNames(referenceRoot) {
return readdirSync(referenceRoot, { withFileTypes: true })
.filter(entry => entry.isDirectory())
.map(entry => entry.name)
.filter(name => name !== '.' && name !== '..')
.sort();
}
const isMainModule = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMainModule) {
processIssues();
}
test/process-parser.test.mjs
+82 -1
View File
@@ -1,6 +1,7 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { readFileSync, readdirSync } from 'node:fs';
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { JSDOM } from 'jsdom';
@@ -10,6 +11,7 @@ globalThis.DOMParser = window.DOMParser;
const { parseIssuesFromHtmlTable } = await import('../web/js/process.js');
const { buildIssueMarkdownDocument } = await import('../web/js/markdown.js');
const { formatFileStatusLine, processIssues } = await import('../reference/process_issues.mjs');
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const fixturesDir = join(__dirname, 'fixtures');
@@ -69,3 +71,82 @@ fixtures.forEach(fixtureId => {
assert.equal(markdown.trim(), expectedMarkdown.trim());
});
});
test('processIssues reports unchanged, modified, and new files distinctly', () => {
const tempRoot = mkdtempSync(join(tmpdir(), 'firewallissues-process-'));
const referenceRoot = join(tempRoot, 'reference');
const outputRoot = join(tempRoot, 'web', 'data', 'issues');
const fixtureRoot = join(fixturesDir, 'PAN-OS-10.2.1-addressed');
const modifiedFixtureRoot = join(fixturesDir, 'PAN-OS-10.1.11-known');
const newFixtureRoot = join(fixturesDir, 'PAN-OS-12.1.2-addressed');
const stdoutChunks = [];
const logs = [];
try {
mkdirSync(join(referenceRoot, 'PAN-OS', 'addressed'), { recursive: true });
mkdirSync(join(referenceRoot, 'PAN-OS', 'known'), { recursive: true });
writeFileSync(
join(referenceRoot, 'PAN-OS', 'addressed', '10.2.1.html'),
readFileSync(join(fixtureRoot, 'input.html'), 'utf-8')
);
writeFileSync(
join(referenceRoot, 'PAN-OS', 'known', '10.1.11.html'),
readFileSync(join(modifiedFixtureRoot, 'input.html'), 'utf-8')
);
writeFileSync(
join(referenceRoot, 'PAN-OS', 'addressed', '12.1.2.html'),
readFileSync(join(newFixtureRoot, 'input.html'), 'utf-8')
);
mkdirSync(join(outputRoot, 'PAN-OS', 'addressed'), { recursive: true });
mkdirSync(join(outputRoot, 'PAN-OS', 'known'), { recursive: true });
writeFileSync(
join(outputRoot, 'PAN-OS', 'addressed', '10.2.1.md'),
readFileSync(join(fixtureRoot, 'expected.md'), 'utf-8')
);
writeFileSync(join(outputRoot, 'PAN-OS', 'known', '10.1.11.md'), 'old content');
const summary = processIssues({
referenceRoot,
outputRoot,
stdout: {
write(chunk) {
stdoutChunks.push(chunk);
}
},
logger: {
log(message) {
logs.push(message);
},
warn(message) {
logs.push(message);
},
error(message) {
logs.push(message);
}
}
});
assert.deepEqual(summary, {
processedProducts: 1,
newCount: 1,
modifiedCount: 1,
noChangeCount: 1,
skippedNoIssues: 0,
});
assert.equal(stdoutChunks.join(''), '\rNo changes for: 1 file\n');
assert.ok(logs.includes(formatFileStatusLine('modified', 'PAN-OS-10.1.11')));
assert.ok(logs.includes(formatFileStatusLine('new', 'PAN-OS-12.1.2')));
assert.equal(
readFileSync(join(outputRoot, 'PAN-OS', 'known', '10.1.11.md'), 'utf-8').trim(),
readFileSync(join(modifiedFixtureRoot, 'expected.md'), 'utf-8').trim()
);
assert.equal(
readFileSync(join(outputRoot, 'PAN-OS', 'addressed', '12.1.2.md'), 'utf-8').trim(),
readFileSync(join(newFixtureRoot, 'expected.md'), 'utf-8').trim()
);
} finally {
rmSync(tempRoot, { recursive: true, force: true });
}
});