From e53a813713fb4f3fbee955b981ff8ca4456286cd Mon Sep 17 00:00:00 2001 From: Aaron Axvig Date: Mon, 13 Apr 2026 11:07:58 -0500 Subject: [PATCH] Cleanup --- README.md | 2 -- scripts/update_products_from_issues.py | 41 ++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 48c65b0..66b92dd 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,6 @@ Known and addressed issues for newly released versions are easily added using th - Fix up the products.json file manually or run `update_products_from_issues.py`. - Submit a pull request. -The date on the end of the Markdown file names is to have some idea of when the data was grabbed. I would imagine that sometimes there are updates. (I plan to remove the dates in the future. The git history should be sufficient.) - There is intentionally no automated scaping of Palo Alto's website, to avoid abuse of server resources. Also releases are not that frequent. A crawler to grab some data from the Common Crawl dataset was started but never really finished. Some data was collected early on when the HTMLTable -> Markdown code was kind of bad, so the formatting of the issue write-up tends to be bad on those. Mostly PAN-OS 10 and 11 stuff. diff --git a/scripts/update_products_from_issues.py b/scripts/update_products_from_issues.py index 872e12a..2008c91 100644 --- a/scripts/update_products_from_issues.py +++ b/scripts/update_products_from_issues.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Update products.json entries from discovered issue data files.""" from __future__ import annotations @@ -18,6 +19,8 @@ HOTFIX_RE = re.compile(r"^(?P\d+(?:\.\d+)*)(?:-h(?P\d+))?$") @dataclass(frozen=True) class FileIdentity: + """Unique identity for a product/version issue bucket.""" + product: str issue_type: str version: str @@ -25,12 +28,16 @@ class FileIdentity: @dataclass(frozen=True) class FileRecord: + """Parsed issue file metadata used for de-duplication and sorting.""" + identity: FileIdentity filename: str release_date: date def parse_args() -> argparse.Namespace: + """Parse command line arguments for input and output paths.""" + parser = argparse.ArgumentParser( description="Update products.json from issue files, keeping latest date per version." ) @@ -48,6 +55,8 @@ def parse_args() -> argparse.Namespace: def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecord]: + """Collect the newest file per product, issue type, and version.""" + latest: dict[FileIdentity, FileRecord] = {} for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()): @@ -78,6 +87,8 @@ def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecor def parse_issue_filename(filename: str) -> tuple[str, date] | None: + """Parse '_.' issue filenames.""" + match = FILENAME_RE.match(filename) if not match: return None @@ -92,6 +103,8 @@ def parse_issue_filename(filename: str) -> tuple[str, date] | None: def clear_leaf_issue_arrays(node: Any) -> None: + """Reset addressed/known arrays on every issue leaf in products.json.""" + if isinstance(node, dict): if "addressed" in node or "known" in node: node["addressed"] = [] @@ -103,10 +116,14 @@ def clear_leaf_issue_arrays(node: Any) -> None: def is_issue_leaf(node: Any) -> bool: + """Return whether a node is a leaf containing issue arrays.""" + return isinstance(node, dict) and ("addressed" in node or "known" in node) def pick_target_leaf_path(products: dict[str, Any], product: str, version: str) -> list[str] | None: + """Choose the best matching product tree path for a version.""" + parts = version.split(".") if not parts: return None @@ -129,6 +146,8 @@ def pick_target_leaf_path(products: dict[str, Any], product: str, version: str) def get_path(data: Any, path: list[str]) -> Any: + """Traverse nested dict keys and return the value at path, if present.""" + current = data for key in path: if not isinstance(current, dict) or key not in current: @@ -138,6 +157,8 @@ def get_path(data: Any, path: list[str]) -> Any: def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]: + """Extract numeric base version parts and optional hotfix number.""" + match = HOTFIX_RE.match(prefix) if not match: return tuple(), -1 @@ -149,13 +170,22 @@ def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]: def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]: + """Build a sort key that orders base versions before hotfixes.""" + prefix = filename.split("_", 1)[0] base_parts, hotfix = parse_release_prefix(prefix) is_hotfix = 1 if hotfix >= 0 else 0 return base_parts, is_hotfix, hotfix, filename -def add_file_to_leaf(products: dict[str, Any], path: list[str], issue_type: str, filename: str) -> None: +def add_file_to_leaf( + products: dict[str, Any], + path: list[str], + issue_type: str, + filename: str, +) -> None: + """Insert an issue filename into the target leaf with stable sorting.""" + leaf = get_path(products, path) if not isinstance(leaf, dict): return @@ -169,6 +199,8 @@ def add_file_to_leaf(products: dict[str, Any], path: list[str], issue_type: str, def update_products(issues_dir: Path, products_path: Path) -> None: + """Rebuild product issue file lists from crawled issue files.""" + if not issues_dir.is_dir(): raise FileNotFoundError(f"Issues directory not found: {issues_dir}") if not products_path.is_file(): @@ -183,7 +215,10 @@ def update_products(issues_dir: Path, products_path: Path) -> None: clear_leaf_issue_arrays(products) latest_files = collect_latest_issue_files(issues_dir) - for record in sorted(latest_files.values(), key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version)): + for record in sorted( + latest_files.values(), + key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version), + ): path = pick_target_leaf_path(products, record.identity.product, record.identity.version) if path is None: continue @@ -195,6 +230,8 @@ def update_products(issues_dir: Path, products_path: Path) -> None: def main() -> int: + """Entrypoint for command-line usage.""" + args = parse_args() script_dir = Path(__file__).resolve().parent repo_root = script_dir.parent