aaron.axvig/firewallissues
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Script cleanup

Browse Source
This commit is contained in:
aaron.axvig
2026-04-13 14:02:13 -05:00
parent 5eed60f8d3
commit ebaa13ca39
2 changed files with 102 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reference/process_issues.mjs
+60 -76
View File
@@ -3,7 +3,7 @@
* Convert downloaded issue table HTML files into web/data/issues/ markdown files. * Convert downloaded issue table HTML files into web/data/issues/ markdown files.
* *
* Usage: * Usage:
* node process_issues.mjs [--crawl CC-MAIN-2026-12] [--product PAN-OS] [--date YYYY-MM-DD] * node reference/process_issues.mjs
* *
* After this script, run: * After this script, run:
* python scripts/update_products_from_issues.py * python scripts/update_products_from_issues.py
@@ -11,125 +11,109 @@
import { JSDOM } from 'jsdom'; import { JSDOM } from 'jsdom';
import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs'; import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
import { join } from 'node:path'; import { join, basename } from 'node:path';
import { fileURLToPath } from 'node:url'; import { fileURLToPath } from 'node:url';
import { parseArgs } from 'node:util';
const __dirname = fileURLToPath(new URL('.', import.meta.url)); const __dirname = fileURLToPath(new URL('.', import.meta.url));
const REPO_ROOT = join(__dirname, '..');
const { values: args } = parseArgs({
options: {
crawl: { type: 'string', default: 'CC-MAIN-2026-12' },
product: { type: 'string', default: 'PAN-OS' },
date: { type: 'string' },
},
strict: false,
});
function isoWeekStartDate(year, week) {
const jan4 = new Date(Date.UTC(year, 0, 4));
const jan4Day = jan4.getUTCDay() || 7;
const week1Monday = new Date(jan4);
week1Monday.setUTCDate(jan4.getUTCDate() - (jan4Day - 1));
const target = new Date(week1Monday);
target.setUTCDate(week1Monday.getUTCDate() + (week - 1) * 7);
return target.toISOString().slice(0, 10);
}
function dateFromCrawlId(crawlId) {
const match = /^CC-MAIN-(\d{4})-(\d{2})$/i.exec(String(crawlId || '').trim());
if (!match) {
return null;
}
const year = Number(match[1]);
const week = Number(match[2]);
if (!Number.isInteger(year) || !Number.isInteger(week) || week < 1 || week > 53) {
return null;
}
return isoWeekStartDate(year, week);
}
const inferredDate = dateFromCrawlId(args.crawl);
const outputDate = args.date || inferredDate || new Date().toISOString().slice(0, 10);
// Set up DOMParser global before importing modules that rely on it. // Set up DOMParser global before importing modules that rely on it.
const { window } = new JSDOM('<!doctype html><html><body></body></html>'); const { window } = new JSDOM('<!doctype html><html><body></body></html>');
globalThis.DOMParser = window.DOMParser; globalThis.DOMParser = window.DOMParser;
const { parseIssuesFromHtmlTable } = await import( const { parseIssuesFromHtmlTable } = await import(
new URL('../../web/js/process.js', import.meta.url).href new URL('../web/js/process.js', import.meta.url).href
); );
const { buildIssueMarkdownDocument } = await import( const { buildIssueMarkdownDocument } = await import(
new URL('../../web/js/markdown.js', import.meta.url).href new URL('../web/js/markdown.js', import.meta.url).href
); );
const REPO_ROOT = join(__dirname, '..', '..'); const OUTPUT_ROOT = join(REPO_ROOT, 'web', 'data', 'issues');
const dataDir = join(__dirname, '..', 'data', args.crawl, args.product);
let files; function readHtmlFiles(dirPath) {
try { try {
files = readdirSync(dataDir).filter(f => f.endsWith('.html')); return readdirSync(dirPath).filter(name => name.endsWith('.html')).sort();
} catch { } catch {
console.error(`No data directory found: ${dataDir}`); return [];
console.error('Run crawl_cc.py first to download issue tables.'); }
process.exit(1);
} }
if (files.length === 0) { function getProductNames() {
console.log(`No HTML files found in ${dataDir}`); return readdirSync(__dirname, { withFileTypes: true })
process.exit(0); .filter(entry => entry.isDirectory())
.map(entry => entry.name)
.filter(name => name !== '.' && name !== '..')
.sort();
}
function getInputEntries(productName) {
const productRoot = join(__dirname, productName);
const entries = [];
for (const issueType of ['addressed', 'known']) {
const issueTypeDir = join(productRoot, issueType);
const files = readHtmlFiles(issueTypeDir);
for (const file of files) {
const version = basename(file, '.html');
entries.push({
filePath: join(issueTypeDir, file),
version,
issueType,
});
}
}
return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
} }
let writtenCount = 0; let writtenCount = 0;
let skippedNoIssues = 0;
let processedProducts = 0;
for (const file of files.sort()) { for (const product of getProductNames()) {
// Filename format: {version}-{type}.html const entries = getInputEntries(product);
// where type is 'addressed' or 'known'. if (entries.length === 0) {
// Version may itself contain hyphens (e.g. '10.2.1-h3'), so split on
// the *last* hyphen-prefixed token that is a known issue type.
const baseName = file.replace(/\.html$/, '');
const lastDash = baseName.lastIndexOf('-');
if (lastDash === -1) {
console.warn(`Skipping unexpected filename: ${file}`);
continue; continue;
} }
const version = baseName.slice(0, lastDash);
const issueType = baseName.slice(lastDash + 1);
if (issueType !== 'addressed' && issueType !== 'known') { processedProducts++;
console.warn(`Skipping unexpected issue type in filename: ${file}`);
continue; for (const entry of entries) {
} const { filePath, issueType, version } = entry;
const capitalizedType = issueType.charAt(0).toUpperCase() + issueType.slice(1); const capitalizedType = issueType.charAt(0).toUpperCase() + issueType.slice(1);
const html = readFileSync(join(dataDir, file), 'utf-8'); const html = readFileSync(filePath, 'utf-8');
const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType }); const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType });
if (parsedIssues.length === 0) { if (parsedIssues.length === 0) {
console.warn(`No issues parsed from ${file} — skipping`); console.warn(`No issues parsed from ${basename(filePath)} — skipping`);
skippedNoIssues++;
continue; continue;
} }
const markdown = buildIssueMarkdownDocument({ const markdown = buildIssueMarkdownDocument({
type: capitalizedType, type: capitalizedType,
product: args.product, product,
version, version,
issues: parsedIssues, issues: parsedIssues,
metadata: {
source: 'common-crawl',
crawl: args.crawl,
},
}); });
const outDir = join(REPO_ROOT, 'web', 'data', 'issues', args.product, issueType); const outDir = join(OUTPUT_ROOT, product, issueType);
mkdirSync(outDir, { recursive: true }); mkdirSync(outDir, { recursive: true });
const outFile = join(outDir, `${version}_${outputDate}.md`);
const outFile = join(outDir, `${version}.md`);
writeFileSync(outFile, markdown, 'utf-8'); writeFileSync(outFile, markdown, 'utf-8');
writtenCount++; writtenCount++;
} }
}
console.log(`Wrote ${writtenCount} markdown file(s) to web/data/issues/${args.product}/`); if (processedProducts === 0) {
console.error(`No HTML files found under ${__dirname}.`);
process.exit(0);
}
console.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`);
if (skippedNoIssues > 0) {
console.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`);
}
if (writtenCount > 0) { if (writtenCount > 0) {
console.log('Next step: python scripts/update_products_from_issues.py'); console.log('Next step: python scripts/update_products_from_issues.py');
} }
scripts/update_products_from_issues.py
+25 -55
View File
@@ -6,40 +6,20 @@ from __future__ import annotations
import argparse import argparse
import json import json
import re import re
from dataclasses import dataclass
from datetime import date
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
ISSUE_TYPES = {"addressed", "known"} ISSUE_TYPES = {"addressed", "known"}
FILENAME_RE = re.compile(r"^(?P<version>[^_]+)_(?P<date>\d{4}-\d{2}-\d{2})\.[^.]+$") VERSION_FILENAME_RE = re.compile(r"^(?P<version>[^.]+(?:\.[^.]+)*)\.md$")
HOTFIX_RE = re.compile(r"^(?P<base>\d+(?:\.\d+)*)(?:-h(?P<hotfix>\d+))?$") HOTFIX_RE = re.compile(r"^(?P<base>\d+(?:\.\d+)*)(?:-h(?P<hotfix>\d+))?$")
@dataclass(frozen=True)
class FileIdentity:
"""Unique identity for a product/version issue bucket."""
product: str
issue_type: str
version: str
@dataclass(frozen=True)
class FileRecord:
"""Parsed issue file metadata used for de-duplication and sorting."""
identity: FileIdentity
filename: str
release_date: date
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
"""Parse command line arguments for input and output paths.""" """Parse command line arguments for input and output paths."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Update products.json from issue files, keeping latest date per version." description="Update products.json from issue files."
) )
parser.add_argument( parser.add_argument(
"--issues-dir", "--issues-dir",
@@ -54,10 +34,10 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args() return parser.parse_args()
def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecord]: def collect_issue_files(issues_dir: Path) -> list[tuple[str, str, str, str]]:
"""Collect the newest file per product, issue type, and version.""" """Collect all issue files with '<version>.md' naming."""
latest: dict[FileIdentity, FileRecord] = {} records: list[tuple[str, str, str, str]] = []
for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()): for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()):
relative = file_path.relative_to(issues_dir) relative = file_path.relative_to(issues_dir)
@@ -68,38 +48,23 @@ def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecor
if issue_type not in ISSUE_TYPES: if issue_type not in ISSUE_TYPES:
continue continue
parsed = parse_issue_filename(filename) version = parse_issue_filename(filename)
if parsed is None: if version is None:
continue continue
version, release_date = parsed records.append((product, issue_type, version, filename))
identity = FileIdentity(product=product, issue_type=issue_type, version=version)
record = FileRecord(identity=identity, filename=filename, release_date=release_date)
current = latest.get(identity) return records
if current is None or (record.release_date, record.filename) > (
current.release_date,
current.filename,
):
latest[identity] = record
return latest
def parse_issue_filename(filename: str) -> tuple[str, date] | None: def parse_issue_filename(filename: str) -> str | None:
"""Parse '<version>_<YYYY-MM-DD>.<ext>' issue filenames.""" """Parse '<version>.md' issue filenames and return version."""
match = FILENAME_RE.match(filename) match = VERSION_FILENAME_RE.match(filename)
if not match: if not match:
return None return None
version = match.group("version") return match.group("version")
try:
release_date = date.fromisoformat(match.group("date"))
except ValueError:
return None
return version, release_date
def clear_leaf_issue_arrays(node: Any) -> None: def clear_leaf_issue_arrays(node: Any) -> None:
@@ -172,7 +137,7 @@ def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]:
def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]: def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]:
"""Build a sort key that orders base versions before hotfixes.""" """Build a sort key that orders base versions before hotfixes."""
prefix = filename.split("_", 1)[0] prefix = filename.rsplit(".", 1)[0]
base_parts, hotfix = parse_release_prefix(prefix) base_parts, hotfix = parse_release_prefix(prefix)
is_hotfix = 1 if hotfix >= 0 else 0 is_hotfix = 1 if hotfix >= 0 else 0
return base_parts, is_hotfix, hotfix, filename return base_parts, is_hotfix, hotfix, filename
@@ -212,17 +177,22 @@ def update_products(issues_dir: Path, products_path: Path) -> None:
if not isinstance(products, dict): if not isinstance(products, dict):
raise ValueError("products.json must contain a top-level object") raise ValueError("products.json must contain a top-level object")
records = collect_issue_files(issues_dir)
if not records:
raise ValueError(
"No '<version>.md' issue files found; refusing to rewrite products.json"
)
clear_leaf_issue_arrays(products) clear_leaf_issue_arrays(products)
latest_files = collect_latest_issue_files(issues_dir) for product, issue_type, version, filename in sorted(
for record in sorted( records,
latest_files.values(), key=lambda r: (r[0], r[1], r[2]),
key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version),
): ):
path = pick_target_leaf_path(products, record.identity.product, record.identity.version) path = pick_target_leaf_path(products, product, version)
if path is None: if path is None:
continue continue
add_file_to_leaf(products, path, record.identity.issue_type, record.filename) add_file_to_leaf(products, path, issue_type, filename)
with products_path.open("w", encoding="utf-8") as f: with products_path.open("w", encoding="utf-8") as f:
json.dump(products, f, indent=2) json.dump(products, f, indent=2)