diff --git a/README.md b/README.md
index 48c65b0..66b92dd 100644
--- a/README.md
+++ b/README.md
@@ -13,8 +13,6 @@ Known and addressed issues for newly released versions are easily added using th
- Fix up the products.json file manually or run `update_products_from_issues.py`.
- Submit a pull request.
-The date on the end of the Markdown file names is to have some idea of when the data was grabbed. I would imagine that sometimes there are updates. (I plan to remove the dates in the future. The git history should be sufficient.)
-
There is intentionally no automated scaping of Palo Alto's website, to avoid abuse of server resources. Also releases are not that frequent. A crawler to grab some data from the Common Crawl dataset was started but never really finished.
Some data was collected early on when the HTMLTable -> Markdown code was kind of bad, so the formatting of the issue write-up tends to be bad on those. Mostly PAN-OS 10 and 11 stuff.
diff --git a/scripts/update_products_from_issues.py b/scripts/update_products_from_issues.py
index 872e12a..2008c91 100644
--- a/scripts/update_products_from_issues.py
+++ b/scripts/update_products_from_issues.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python3
+"""Update products.json entries from discovered issue data files."""
from __future__ import annotations
@@ -18,6 +19,8 @@ HOTFIX_RE = re.compile(r"^(?P\d+(?:\.\d+)*)(?:-h(?P\d+))?$")
@dataclass(frozen=True)
class FileIdentity:
+ """Unique identity for a product/version issue bucket."""
+
product: str
issue_type: str
version: str
@@ -25,12 +28,16 @@ class FileIdentity:
@dataclass(frozen=True)
class FileRecord:
+ """Parsed issue file metadata used for de-duplication and sorting."""
+
identity: FileIdentity
filename: str
release_date: date
def parse_args() -> argparse.Namespace:
+ """Parse command line arguments for input and output paths."""
+
parser = argparse.ArgumentParser(
description="Update products.json from issue files, keeping latest date per version."
)
@@ -48,6 +55,8 @@ def parse_args() -> argparse.Namespace:
def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecord]:
+ """Collect the newest file per product, issue type, and version."""
+
latest: dict[FileIdentity, FileRecord] = {}
for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()):
@@ -78,6 +87,8 @@ def collect_latest_issue_files(issues_dir: Path) -> dict[FileIdentity, FileRecor
def parse_issue_filename(filename: str) -> tuple[str, date] | None:
+ """Parse '_.' issue filenames."""
+
match = FILENAME_RE.match(filename)
if not match:
return None
@@ -92,6 +103,8 @@ def parse_issue_filename(filename: str) -> tuple[str, date] | None:
def clear_leaf_issue_arrays(node: Any) -> None:
+ """Reset addressed/known arrays on every issue leaf in products.json."""
+
if isinstance(node, dict):
if "addressed" in node or "known" in node:
node["addressed"] = []
@@ -103,10 +116,14 @@ def clear_leaf_issue_arrays(node: Any) -> None:
def is_issue_leaf(node: Any) -> bool:
+ """Return whether a node is a leaf containing issue arrays."""
+
return isinstance(node, dict) and ("addressed" in node or "known" in node)
def pick_target_leaf_path(products: dict[str, Any], product: str, version: str) -> list[str] | None:
+ """Choose the best matching product tree path for a version."""
+
parts = version.split(".")
if not parts:
return None
@@ -129,6 +146,8 @@ def pick_target_leaf_path(products: dict[str, Any], product: str, version: str)
def get_path(data: Any, path: list[str]) -> Any:
+ """Traverse nested dict keys and return the value at path, if present."""
+
current = data
for key in path:
if not isinstance(current, dict) or key not in current:
@@ -138,6 +157,8 @@ def get_path(data: Any, path: list[str]) -> Any:
def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]:
+ """Extract numeric base version parts and optional hotfix number."""
+
match = HOTFIX_RE.match(prefix)
if not match:
return tuple(), -1
@@ -149,13 +170,22 @@ def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]:
def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]:
+ """Build a sort key that orders base versions before hotfixes."""
+
prefix = filename.split("_", 1)[0]
base_parts, hotfix = parse_release_prefix(prefix)
is_hotfix = 1 if hotfix >= 0 else 0
return base_parts, is_hotfix, hotfix, filename
-def add_file_to_leaf(products: dict[str, Any], path: list[str], issue_type: str, filename: str) -> None:
+def add_file_to_leaf(
+ products: dict[str, Any],
+ path: list[str],
+ issue_type: str,
+ filename: str,
+) -> None:
+ """Insert an issue filename into the target leaf with stable sorting."""
+
leaf = get_path(products, path)
if not isinstance(leaf, dict):
return
@@ -169,6 +199,8 @@ def add_file_to_leaf(products: dict[str, Any], path: list[str], issue_type: str,
def update_products(issues_dir: Path, products_path: Path) -> None:
+ """Rebuild product issue file lists from crawled issue files."""
+
if not issues_dir.is_dir():
raise FileNotFoundError(f"Issues directory not found: {issues_dir}")
if not products_path.is_file():
@@ -183,7 +215,10 @@ def update_products(issues_dir: Path, products_path: Path) -> None:
clear_leaf_issue_arrays(products)
latest_files = collect_latest_issue_files(issues_dir)
- for record in sorted(latest_files.values(), key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version)):
+ for record in sorted(
+ latest_files.values(),
+ key=lambda r: (r.identity.product, r.identity.issue_type, r.identity.version),
+ ):
path = pick_target_leaf_path(products, record.identity.product, record.identity.version)
if path is None:
continue
@@ -195,6 +230,8 @@ def update_products(issues_dir: Path, products_path: Path) -> None:
def main() -> int:
+ """Entrypoint for command-line usage."""
+
args = parse_args()
script_dir = Path(__file__).resolve().parent
repo_root = script_dir.parent