aaron.axvig/firewallissues
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Script cleanup

Browse Source
This commit is contained in:
aaron.axvig
2026-04-13 14:02:13 -05:00
parent 5eed60f8d3
commit ebaa13ca39
2 changed files with 102 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reference/process_issues.mjs
+60 -76
View File
@@ -3,7 +3,7 @@
* Convert downloaded issue table HTML files into web/data/issues/ markdown files.
*
* Usage:
* node process_issues.mjs [--crawl CC-MAIN-2026-12] [--product PAN-OS] [--date YYYY-MM-DD]
* node reference/process_issues.mjs
*
* After this script, run:
* python scripts/update_products_from_issues.py
@@ -11,125 +11,109 @@
import { JSDOM } from 'jsdom';
import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
import { join } from 'node:path';
import { join, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseArgs } from 'node:util';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const { values: args } = parseArgs({
options: {
crawl: { type: 'string', default: 'CC-MAIN-2026-12' },
product: { type: 'string', default: 'PAN-OS' },
date: { type: 'string' },
},
strict: false,
});
function isoWeekStartDate(year, week) {
const jan4 = new Date(Date.UTC(year, 0, 4));
const jan4Day = jan4.getUTCDay() || 7;
const week1Monday = new Date(jan4);
week1Monday.setUTCDate(jan4.getUTCDate() - (jan4Day - 1));
const target = new Date(week1Monday);
target.setUTCDate(week1Monday.getUTCDate() + (week - 1) * 7);
return target.toISOString().slice(0, 10);
}
function dateFromCrawlId(crawlId) {
const match = /^CC-MAIN-(\d{4})-(\d{2})$/i.exec(String(crawlId || '').trim());
if (!match) {
return null;
}
const year = Number(match[1]);
const week = Number(match[2]);
if (!Number.isInteger(year) || !Number.isInteger(week) || week < 1 || week > 53) {
return null;
}
return isoWeekStartDate(year, week);
}
const inferredDate = dateFromCrawlId(args.crawl);
const outputDate = args.date || inferredDate || new Date().toISOString().slice(0, 10);
const REPO_ROOT = join(__dirname, '..');
// Set up DOMParser global before importing modules that rely on it.
const { window } = new JSDOM('<!doctype html><html><body></body></html>');
globalThis.DOMParser = window.DOMParser;
const { parseIssuesFromHtmlTable } = await import(
new URL('../../web/js/process.js', import.meta.url).href
new URL('../web/js/process.js', import.meta.url).href
);
const { buildIssueMarkdownDocument } = await import(
new URL('../../web/js/markdown.js', import.meta.url).href
new URL('../web/js/markdown.js', import.meta.url).href
);
const REPO_ROOT = join(__dirname, '..', '..');
const dataDir = join(__dirname, '..', 'data', args.crawl, args.product);
const OUTPUT_ROOT = join(REPO_ROOT, 'web', 'data', 'issues');
let files;
function readHtmlFiles(dirPath) {
try {
files = readdirSync(dataDir).filter(f => f.endsWith('.html'));
return readdirSync(dirPath).filter(name => name.endsWith('.html')).sort();
} catch {
console.error(`No data directory found: ${dataDir}`);
console.error('Run crawl_cc.py first to download issue tables.');
process.exit(1);
return [];
}
}
if (files.length === 0) {
console.log(`No HTML files found in ${dataDir}`);
process.exit(0);
function getProductNames() {
return readdirSync(__dirname, { withFileTypes: true })
.filter(entry => entry.isDirectory())
.map(entry => entry.name)
.filter(name => name !== '.' && name !== '..')
.sort();
}
function getInputEntries(productName) {
const productRoot = join(__dirname, productName);
const entries = [];
for (const issueType of ['addressed', 'known']) {
const issueTypeDir = join(productRoot, issueType);
const files = readHtmlFiles(issueTypeDir);
for (const file of files) {
const version = basename(file, '.html');
entries.push({
filePath: join(issueTypeDir, file),
version,
issueType,
});
}
}
return entries.sort((a, b) => a.filePath.localeCompare(b.filePath));
}
let writtenCount = 0;
let skippedNoIssues = 0;
let processedProducts = 0;
for (const file of files.sort()) {
// Filename format: {version}-{type}.html
// where type is 'addressed' or 'known'.
// Version may itself contain hyphens (e.g. '10.2.1-h3'), so split on
// the *last* hyphen-prefixed token that is a known issue type.
const baseName = file.replace(/\.html$/, '');
const lastDash = baseName.lastIndexOf('-');
if (lastDash === -1) {
console.warn(`Skipping unexpected filename: ${file}`);
for (const product of getProductNames()) {
const entries = getInputEntries(product);
if (entries.length === 0) {
continue;
}
const version = baseName.slice(0, lastDash);
const issueType = baseName.slice(lastDash + 1);
if (issueType !== 'addressed' && issueType !== 'known') {
console.warn(`Skipping unexpected issue type in filename: ${file}`);
continue;
}
processedProducts++;
for (const entry of entries) {
const { filePath, issueType, version } = entry;
const capitalizedType = issueType.charAt(0).toUpperCase() + issueType.slice(1);
const html = readFileSync(join(dataDir, file), 'utf-8');
const html = readFileSync(filePath, 'utf-8');
const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType });
if (parsedIssues.length === 0) {
console.warn(`No issues parsed from ${file} — skipping`);
console.warn(`No issues parsed from ${basename(filePath)} — skipping`);
skippedNoIssues++;
continue;
}
const markdown = buildIssueMarkdownDocument({
type: capitalizedType,
product: args.product,
product,
version,
issues: parsedIssues,
metadata: {
source: 'common-crawl',
crawl: args.crawl,
},
});
const outDir = join(REPO_ROOT, 'web', 'data', 'issues', args.product, issueType);
const outDir = join(OUTPUT_ROOT, product, issueType);
mkdirSync(outDir, { recursive: true });
const outFile = join(outDir, `${version}_${outputDate}.md`);
const outFile = join(outDir, `${version}.md`);
writeFileSync(outFile, markdown, 'utf-8');
writtenCount++;
}
}
console.log(`Wrote ${writtenCount} markdown file(s) to web/data/issues/${args.product}/`);
if (processedProducts === 0) {
console.error(`No HTML files found under ${__dirname}.`);
process.exit(0);
}
console.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`);
if (skippedNoIssues > 0) {
console.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`);
}
if (writtenCount > 0) {
console.log('Next step: python scripts/update_products_from_issues.py');
}
scripts/update_products_from_issues.py
+25 -55
View File
@@ -6,40 +6,20 @@ from __future__ import annotations
import argparse
import json
import re
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from typing import Any
ISSUE_TYPES = {"addressed", "known"}
FILENAME_RE = re.compile(r"^(?P<version>[^_]+)_(?P<date>\d{4}-\d{2}-\d{2})\.[^.]+$")
VERSION_FILENAME_RE = re.compile(r"^(?P<version>[^.]+(?:\.[^.]+)*)\.md$")
HOTFIX_RE = re.compile(r"^(?P<base>\d+(?:\.\d+)*)(?:-h(?P<hotfix>\d+))?$")
@dataclass(frozen=True)
class FileIdentity:
"""Unique identity for a product/version issue bucket."""
product: str
issue_type: str
version: str
@dataclass(frozen=True)
class FileRecord:
"""Parsed issue file metadata used for de-duplication and sorting."""
identity: FileIdentity
filename: str
release_date: date
def parse_args() -> argparse.Namespace:
"""Parse command line arguments for input and output paths."""
parser = argparse.ArgumentParser(
description="Update products.json from issue files, keeping latest date per version."
description="Update products.json from issue files."
)
parser.add_argument(
"--issues-dir",
@@ -54,10 +34,10 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecord]:
"""Collect the newest file per product, issue type, and version."""
def collect_issue_files(issues_dir: Path) -> list[tuple[str, str, str, str]]:
"""Collect all issue files with '<version>.md' naming."""
latest: dict[FileIdentity, FileRecord] = {}
records: list[tuple[str, str, str, str]] = []
for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()):
relative = file_path.relative_to(issues_dir)
@@ -68,38 +48,23 @@ def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecor
if issue_type not in ISSUE_TYPES:
continue
parsed = parse_issue_filename(filename)
if parsed is None:
version = parse_issue_filename(filename)
if version is None:
continue
version, release_date = parsed
identity = FileIdentity(product=product, issue_type=issue_type, version=version)
record = FileRecord(identity=identity, filename=filename, release_date=release_date)
records.append((product, issue_type, version, filename))
current = latest.get(identity)
if current is None or (record.release_date, record.filename) > (
current.release_date,
current.filename,
):
latest[identity] = record
return latest
return records
def parse_issue_filename(filename: str) -> tuple[str, date] | None:
"""Parse '<version>_<YYYY-MM-DD>.<ext>' issue filenames."""
def parse_issue_filename(filename: str) -> str | None:
"""Parse '<version>.md' issue filenames and return version."""
match = FILENAME_RE.match(filename)
match = VERSION_FILENAME_RE.match(filename)
if not match:
return None
version = match.group("version")
try:
release_date = date.fromisoformat(match.group("date"))
except ValueError:
return None
return version, release_date
return match.group("version")
def clear_leaf_issue_arrays(node: Any) -> None:
@@ -172,7 +137,7 @@ def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]:
def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]:
"""Build a sort key that orders base versions before hotfixes."""
prefix = filename.split("_", 1)[0]
prefix = filename.rsplit(".", 1)[0]
base_parts, hotfix = parse_release_prefix(prefix)
is_hotfix = 1 if hotfix >= 0 else 0
return base_parts, is_hotfix, hotfix, filename
@@ -212,17 +177,22 @@ def update_products(issues_dir: Path, products_path: Path) -> None:
if not isinstance(products, dict):
raise ValueError("products.json must contain a top-level object")
records = collect_issue_files(issues_dir)
if not records:
raise ValueError(
"No '<version>.md' issue files found; refusing to rewrite products.json"
)
clear_leaf_issue_arrays(products)
latest_files = collect_latest_issue_files(issues_dir)
for record in sorted(
latest_files.values(),
key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version),
for product, issue_type, version, filename in sorted(
records,
key=lambda r: (r[0], r[1], r[2]),
):
path = pick_target_leaf_path(products, record.identity.product, record.identity.version)
path = pick_target_leaf_path(products, product, version)
if path is None:
continue
add_file_to_leaf(products, path, record.identity.issue_type, record.filename)
add_file_to_leaf(products, path, issue_type, filename)
with products_path.open("w", encoding="utf-8") as f:
json.dump(products, f, indent=2)