diff --git a/reference/process_issues.mjs b/reference/process_issues.mjs index 02c467e..346d080 100644 --- a/reference/process_issues.mjs +++ b/reference/process_issues.mjs @@ -3,7 +3,7 @@ * Convert downloaded issue table HTML files into web/data/issues/ markdown files. * * Usage: - * node process_issues.mjs [--crawl CC-MAIN-2026-12] [--product PAN-OS] [--date YYYY-MM-DD] + * node reference/process_issues.mjs * * After this script, run: * python scripts/update_products_from_issues.py @@ -11,125 +11,109 @@ import { JSDOM } from 'jsdom'; import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs'; -import { join } from 'node:path'; +import { join, basename } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { parseArgs } from 'node:util'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); - -const { values: args } = parseArgs({ - options: { - crawl: { type: 'string', default: 'CC-MAIN-2026-12' }, - product: { type: 'string', default: 'PAN-OS' }, - date: { type: 'string' }, - }, - strict: false, -}); - -function isoWeekStartDate(year, week) { - const jan4 = new Date(Date.UTC(year, 0, 4)); - const jan4Day = jan4.getUTCDay() || 7; - const week1Monday = new Date(jan4); - week1Monday.setUTCDate(jan4.getUTCDate() - (jan4Day - 1)); - - const target = new Date(week1Monday); - target.setUTCDate(week1Monday.getUTCDate() + (week - 1) * 7); - return target.toISOString().slice(0, 10); -} - -function dateFromCrawlId(crawlId) { - const match = /^CC-MAIN-(\d{4})-(\d{2})$/i.exec(String(crawlId || '').trim()); - if (!match) { - return null; - } - const year = Number(match[1]); - const week = Number(match[2]); - if (!Number.isInteger(year) || !Number.isInteger(week) || week < 1 || week > 53) { - return null; - } - return isoWeekStartDate(year, week); -} - -const inferredDate = dateFromCrawlId(args.crawl); -const outputDate = args.date || inferredDate || new Date().toISOString().slice(0, 10); +const REPO_ROOT = join(__dirname, '..'); // Set up DOMParser global before importing modules that rely on it. const { window } = new JSDOM(''); globalThis.DOMParser = window.DOMParser; const { parseIssuesFromHtmlTable } = await import( - new URL('../../web/js/process.js', import.meta.url).href + new URL('../web/js/process.js', import.meta.url).href ); const { buildIssueMarkdownDocument } = await import( - new URL('../../web/js/markdown.js', import.meta.url).href + new URL('../web/js/markdown.js', import.meta.url).href ); -const REPO_ROOT = join(__dirname, '..', '..'); -const dataDir = join(__dirname, '..', 'data', args.crawl, args.product); +const OUTPUT_ROOT = join(REPO_ROOT, 'web', 'data', 'issues'); -let files; -try { - files = readdirSync(dataDir).filter(f => f.endsWith('.html')); -} catch { - console.error(`No data directory found: ${dataDir}`); - console.error('Run crawl_cc.py first to download issue tables.'); - process.exit(1); +function readHtmlFiles(dirPath) { + try { + return readdirSync(dirPath).filter(name => name.endsWith('.html')).sort(); + } catch { + return []; + } } -if (files.length === 0) { - console.log(`No HTML files found in ${dataDir}`); - process.exit(0); +function getProductNames() { + return readdirSync(__dirname, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .filter(name => name !== '.' && name !== '..') + .sort(); +} + +function getInputEntries(productName) { + const productRoot = join(__dirname, productName); + const entries = []; + for (const issueType of ['addressed', 'known']) { + const issueTypeDir = join(productRoot, issueType); + const files = readHtmlFiles(issueTypeDir); + for (const file of files) { + const version = basename(file, '.html'); + entries.push({ + filePath: join(issueTypeDir, file), + version, + issueType, + }); + } + } + return entries.sort((a, b) => a.filePath.localeCompare(b.filePath)); } let writtenCount = 0; +let skippedNoIssues = 0; +let processedProducts = 0; -for (const file of files.sort()) { - // Filename format: {version}-{type}.html - // where type is 'addressed' or 'known'. - // Version may itself contain hyphens (e.g. '10.2.1-h3'), so split on - // the *last* hyphen-prefixed token that is a known issue type. - const baseName = file.replace(/\.html$/, ''); - const lastDash = baseName.lastIndexOf('-'); - if (lastDash === -1) { - console.warn(`Skipping unexpected filename: ${file}`); - continue; - } - const version = baseName.slice(0, lastDash); - const issueType = baseName.slice(lastDash + 1); - - if (issueType !== 'addressed' && issueType !== 'known') { - console.warn(`Skipping unexpected issue type in filename: ${file}`); +for (const product of getProductNames()) { + const entries = getInputEntries(product); + if (entries.length === 0) { continue; } - const capitalizedType = issueType.charAt(0).toUpperCase() + issueType.slice(1); - const html = readFileSync(join(dataDir, file), 'utf-8'); + processedProducts++; - const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType }); - if (parsedIssues.length === 0) { - console.warn(`No issues parsed from ${file} — skipping`); - continue; + for (const entry of entries) { + const { filePath, issueType, version } = entry; + + const capitalizedType = issueType.charAt(0).toUpperCase() + issueType.slice(1); + const html = readFileSync(filePath, 'utf-8'); + + const parsedIssues = parseIssuesFromHtmlTable(html, { type: capitalizedType }); + if (parsedIssues.length === 0) { + console.warn(`No issues parsed from ${basename(filePath)} — skipping`); + skippedNoIssues++; + continue; + } + + const markdown = buildIssueMarkdownDocument({ + type: capitalizedType, + product, + version, + issues: parsedIssues, + }); + + const outDir = join(OUTPUT_ROOT, product, issueType); + mkdirSync(outDir, { recursive: true }); + + const outFile = join(outDir, `${version}.md`); + writeFileSync(outFile, markdown, 'utf-8'); + writtenCount++; } - - const markdown = buildIssueMarkdownDocument({ - type: capitalizedType, - product: args.product, - version, - issues: parsedIssues, - metadata: { - source: 'common-crawl', - crawl: args.crawl, - }, - }); - - const outDir = join(REPO_ROOT, 'web', 'data', 'issues', args.product, issueType); - mkdirSync(outDir, { recursive: true }); - const outFile = join(outDir, `${version}_${outputDate}.md`); - writeFileSync(outFile, markdown, 'utf-8'); - writtenCount++; } -console.log(`Wrote ${writtenCount} markdown file(s) to web/data/issues/${args.product}/`); +if (processedProducts === 0) { + console.error(`No HTML files found under ${__dirname}.`); + process.exit(0); +} + +console.log(`Wrote ${writtenCount} markdown file(s) across ${processedProducts} product(s).`); +if (skippedNoIssues > 0) { + console.log(`Skipped ${skippedNoIssues} file(s) with no parsed issues.`); +} if (writtenCount > 0) { console.log('Next step: python scripts/update_products_from_issues.py'); } diff --git a/scripts/update_products_from_issues.py b/scripts/update_products_from_issues.py index 2008c91..754c286 100644 --- a/scripts/update_products_from_issues.py +++ b/scripts/update_products_from_issues.py @@ -6,40 +6,20 @@ from __future__ import annotations import argparse import json import re -from dataclasses import dataclass -from datetime import date from pathlib import Path from typing import Any ISSUE_TYPES = {"addressed", "known"} -FILENAME_RE = re.compile(r"^(?P[^_]+)_(?P\d{4}-\d{2}-\d{2})\.[^.]+$") +VERSION_FILENAME_RE = re.compile(r"^(?P[^.]+(?:\.[^.]+)*)\.md$") HOTFIX_RE = re.compile(r"^(?P\d+(?:\.\d+)*)(?:-h(?P\d+))?$") -@dataclass(frozen=True) -class FileIdentity: - """Unique identity for a product/version issue bucket.""" - - product: str - issue_type: str - version: str - - -@dataclass(frozen=True) -class FileRecord: - """Parsed issue file metadata used for de-duplication and sorting.""" - - identity: FileIdentity - filename: str - release_date: date - - def parse_args() -> argparse.Namespace: """Parse command line arguments for input and output paths.""" parser = argparse.ArgumentParser( - description="Update products.json from issue files, keeping latest date per version." + description="Update products.json from issue files." ) parser.add_argument( "--issues-dir", @@ -54,10 +34,10 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecord]: - """Collect the newest file per product, issue type, and version.""" +def collect_issue_files(issues_dir: Path) -> list[tuple[str, str, str, str]]: + """Collect all issue files with '.md' naming.""" - latest: dict[FileIdentity, FileRecord] = {} + records: list[tuple[str, str, str, str]] = [] for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()): relative = file_path.relative_to(issues_dir) @@ -68,38 +48,23 @@ def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecor if issue_type not in ISSUE_TYPES: continue - parsed = parse_issue_filename(filename) - if parsed is None: + version = parse_issue_filename(filename) + if version is None: continue - version, release_date = parsed - identity = FileIdentity(product=product, issue_type=issue_type, version=version) - record = FileRecord(identity=identity, filename=filename, release_date=release_date) + records.append((product, issue_type, version, filename)) - current = latest.get(identity) - if current is None or (record.release_date, record.filename) > ( - current.release_date, - current.filename, - ): - latest[identity] = record - - return latest + return records -def parse_issue_filename(filename: str) -> tuple[str, date] | None: - """Parse '_.' issue filenames.""" +def parse_issue_filename(filename: str) -> str | None: + """Parse '.md' issue filenames and return version.""" - match = FILENAME_RE.match(filename) + match = VERSION_FILENAME_RE.match(filename) if not match: return None - version = match.group("version") - try: - release_date = date.fromisoformat(match.group("date")) - except ValueError: - return None - - return version, release_date + return match.group("version") def clear_leaf_issue_arrays(node: Any) -> None: @@ -172,7 +137,7 @@ def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]: def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]: """Build a sort key that orders base versions before hotfixes.""" - prefix = filename.split("_", 1)[0] + prefix = filename.rsplit(".", 1)[0] base_parts, hotfix = parse_release_prefix(prefix) is_hotfix = 1 if hotfix >= 0 else 0 return base_parts, is_hotfix, hotfix, filename @@ -212,17 +177,22 @@ def update_products(issues_dir: Path, products_path: Path) -> None: if not isinstance(products, dict): raise ValueError("products.json must contain a top-level object") + records = collect_issue_files(issues_dir) + if not records: + raise ValueError( + "No '.md' issue files found; refusing to rewrite products.json" + ) + clear_leaf_issue_arrays(products) - latest_files = collect_latest_issue_files(issues_dir) - for record in sorted( - latest_files.values(), - key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version), + for product, issue_type, version, filename in sorted( + records, + key=lambda r: (r[0], r[1], r[2]), ): - path = pick_target_leaf_path(products, record.identity.product, record.identity.version) + path = pick_target_leaf_path(products, product, version) if path is None: continue - add_file_to_leaf(products, path, record.identity.issue_type, record.filename) + add_file_to_leaf(products, path, issue_type, filename) with products_path.open("w", encoding="utf-8") as f: json.dump(products, f, indent=2)