diff --git a/README.md b/README.md index 66b92dd..7170d8d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ Known and addressed issues for newly released versions are easily added using th - Copy the issue table's HTML from the webpage using devtools. - Fill out the process.html page's fields and paste in the table HTML. - Download the Markdown file and put it into the correct folder. -- Fix up the products.json file manually or run `update_products_from_issues.py`. +- Run `npm run update:generated` to update `products.json`, append new stable + product-tree manifest slots, and regenerate rendered test fixtures. - Submit a pull request. There is intentionally no automated scaping of Palo Alto's website, to avoid abuse of server resources. Also releases are not that frequent. A crawler to grab some data from the Common Crawl dataset was started but never really finished. diff --git a/package.json b/package.json index daa860a..4008ba1 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "type": "module", "scripts": { "test": "node --test", + "update:products": "node scripts/update-products-from-issues.mjs", + "update:fixtures": "node scripts/generate-rendered-fixtures.mjs", + "update:generated": "node scripts/refresh-generated-data.mjs", "vendor:marked": "mkdir -p web/vendor && cp node_modules/marked/lib/marked.esm.js web/vendor/marked.esm.js" }, "devDependencies": { diff --git a/scripts/generate-rendered-fixtures.mjs b/scripts/generate-rendered-fixtures.mjs index 82d9cda..75b2243 100644 --- a/scripts/generate-rendered-fixtures.mjs +++ b/scripts/generate-rendered-fixtures.mjs @@ -1,11 +1,11 @@ /** * Generates expected-parsed.json for each fixture directory that contains expected.md. - * Run manually: node test/generate-rendered-fixtures.mjs + * Run manually: node scripts/generate-rendered-fixtures.mjs */ import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { JSDOM } from 'jsdom'; const { window } = new JSDOM(''); @@ -20,31 +20,38 @@ globalThis.fetch = () => Promise.resolve({ const { parseMarkdownIssues, markdownSummaryToHtml } = await import('../web/js/issues.js'); const __dirname = fileURLToPath(new URL('.', import.meta.url)); -const fixturesDir = join(__dirname, '../test/fixtures'); +const defaultFixturesDir = join(__dirname, '../test/fixtures'); -const fixtures = readdirSync(fixturesDir).filter(name => { - const contents = readdirSync(join(fixturesDir, name)).map(d => d); - return contents.includes('expected.md'); -}); +export function generateRenderedFixtures(fixturesDir = defaultFixturesDir) { + const fixtures = readdirSync(fixturesDir).filter(name => { + const contents = readdirSync(join(fixturesDir, name)).map(d => d); + return contents.includes('expected.md'); + }); -for (const fixtureId of fixtures) { - const fixturePath = join(fixturesDir, fixtureId); - const markdownText = readFileSync(join(fixturePath, 'expected.md'), 'utf-8'); - const parsed = parseMarkdownIssues(markdownText); + for (const fixtureId of fixtures) { + const fixturePath = join(fixturesDir, fixtureId); + const markdownText = readFileSync(join(fixturePath, 'expected.md'), 'utf-8'); + const parsed = parseMarkdownIssues(markdownText); - const result = parsed.map(issue => ({ - id: issue.id, - summary: issue.summary, - resolved: issue.resolved, - caveat: issue.caveat, - renderedHtml: markdownSummaryToHtml(issue.summary, issue.resolved, issue.caveat) - })); + const result = parsed.map(issue => ({ + id: issue.id, + summary: issue.summary, + resolved: issue.resolved, + caveat: issue.caveat, + renderedHtml: markdownSummaryToHtml(issue.summary, issue.resolved, issue.caveat) + })); - writeFileSync( - join(fixturePath, 'expected-parsed.json'), - JSON.stringify(result, null, 2) + '\n', - 'utf-8' - ); + writeFileSync( + join(fixturePath, 'expected-parsed.json'), + JSON.stringify(result, null, 2) + '\n', + 'utf-8' + ); - console.log(`Written ${fixtureId}/expected-parsed.json (${result.length} issues)`); + console.log(`Written ${fixtureId}/expected-parsed.json (${result.length} issues)`); + } + return { fixtures: fixtures.length }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + generateRenderedFixtures(); } diff --git a/scripts/refresh-generated-data.mjs b/scripts/refresh-generated-data.mjs new file mode 100644 index 0000000..eebe1e1 --- /dev/null +++ b/scripts/refresh-generated-data.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { runUpdateProductsCli } from './update-products-from-issues.mjs'; +import { generateRenderedFixtures } from './generate-rendered-fixtures.mjs'; + +await runUpdateProductsCli(); +const fixtureResult = generateRenderedFixtures(); +console.log(`Regenerated ${fixtureResult.fixtures} rendered fixture set(s).`); diff --git a/scripts/update-products-from-issues.mjs b/scripts/update-products-from-issues.mjs new file mode 100644 index 0000000..87e0aff --- /dev/null +++ b/scripts/update-products-from-issues.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +import { readdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { buildProductTree, enumerateTreePaths } from '../web/js/product-tree-model.js'; + +const ISSUE_TYPES = new Set(['addressed', 'known']); +const VERSION_FILENAME_RE = /^(?[^.]+(?:\.[^.]+)*)\.md$/; +const HOTFIX_RE = /^(?\d+(?:\.\d+)*)(?:-h(?\d+))?$/; +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + +async function walkFiles(root) { + const files = []; + async function visit(directory) { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push(path); + } + } + await visit(root); + return files.sort(); +} + +export async function collectIssueFiles(issuesDir) { + const records = []; + for (const filePath of await walkFiles(issuesDir)) { + const parts = relative(issuesDir, filePath).split(/[\\/]/); + if (parts.length !== 3) continue; + const [product, issueType, filename] = parts; + if (!ISSUE_TYPES.has(issueType)) continue; + const match = VERSION_FILENAME_RE.exec(filename); + if (!match) continue; + records.push({ product, issueType, version: match.groups.version, filename }); + } + return records; +} + +function clearLeafIssueArrays(node) { + if (!node || typeof node !== 'object' || Array.isArray(node)) return; + if ('addressed' in node || 'known' in node) { + node.addressed = []; + node.known = []; + return; + } + Object.values(node).forEach(clearLeafIssueArrays); +} + +function getPath(data, path) { + let current = data; + for (const key of path) { + if (!current || typeof current !== 'object' || !(key in current)) return null; + current = current[key]; + } + return current; +} + +function isIssueLeaf(node) { + return Boolean(node) && typeof node === 'object' && ('addressed' in node || 'known' in node); +} + +function pickTargetLeafPath(products, product, version) { + const parts = version.split('.'); + const major = parts[0]; + const minor = parts.length > 1 ? `${parts[0]}.${parts[1]}` : major; + for (const path of [[product, major, minor], [product, major], [product]]) { + if (isIssueLeaf(getPath(products, path))) return path; + } + return null; +} + +function parseReleasePrefix(prefix) { + const match = HOTFIX_RE.exec(prefix); + if (!match) return { base: [], hotfix: -1 }; + return { + base: match.groups.base.split('.').map(Number), + hotfix: match.groups.hotfix === undefined ? -1 : Number(match.groups.hotfix) + }; +} + +function compareNumberArrays(left, right) { + const length = Math.max(left.length, right.length); + for (let index = 0; index < length; index++) { + const difference = (left[index] ?? -1) - (right[index] ?? -1); + if (difference) return difference; + } + return 0; +} + +function compareIssueFilenames(left, right) { + const leftPrefix = left.replace(/\.[^.]+$/, ''); + const rightPrefix = right.replace(/\.[^.]+$/, ''); + const leftVersion = parseReleasePrefix(leftPrefix); + const rightVersion = parseReleasePrefix(rightPrefix); + return compareNumberArrays(leftVersion.base, rightVersion.base) || + Number(leftVersion.hotfix >= 0) - Number(rightVersion.hotfix >= 0) || + leftVersion.hotfix - rightVersion.hotfix || + left.localeCompare(right); +} + +export function rebuildProducts(products, records) { + clearLeafIssueArrays(products); + const unmatched = []; + records.sort((left, right) => + left.product.localeCompare(right.product) || + left.issueType.localeCompare(right.issueType) || + left.version.localeCompare(right.version) + ); + + for (const record of records) { + const path = pickTargetLeafPath(products, record.product, record.version); + if (!path) { + unmatched.push(record); + continue; + } + const leaf = getPath(products, path); + const files = Array.isArray(leaf[record.issueType]) ? leaf[record.issueType] : []; + leaf[record.issueType] = Array.from(new Set([...files, record.filename])).sort(compareIssueFilenames); + } + return { products, unmatched }; +} + +export function updateManifest(manifest, currentPaths) { + if (!manifest || manifest.schema !== 1 || !Array.isArray(manifest.nodes)) { + throw new Error('Product-tree manifest must use schema 1 with a nodes array.'); + } + const active = new Set(); + manifest.nodes.forEach((path, slot) => { + if (path === null) return; + if (typeof path !== 'string' || !path) throw new Error(`Invalid manifest entry at slot ${slot}.`); + if (active.has(path)) throw new Error(`Duplicate manifest path: ${path}`); + active.add(path); + }); + + const current = new Set(currentPaths); + const appended = currentPaths.filter(path => !active.has(path)); + manifest.nodes.push(...appended); + const missing = Array.from(active).filter(path => !current.has(path)); + return { manifest, appended, missing }; +} + +export async function updateProductsFromIssues({ issuesDir, productsPath, manifestPath }) { + const [productsText, manifestText, records] = await Promise.all([ + readFile(productsPath, 'utf8'), + readFile(manifestPath, 'utf8'), + collectIssueFiles(issuesDir) + ]); + if (records.length === 0) { + throw new Error("No '.md' issue files found; refusing to rewrite products.json"); + } + const products = JSON.parse(productsText); + if (!products || typeof products !== 'object' || Array.isArray(products)) { + throw new Error('products.json must contain a top-level object.'); + } + + const rebuilt = rebuildProducts(products, records); + const paths = enumerateTreePaths(buildProductTree(rebuilt.products)); + const manifestUpdate = updateManifest(JSON.parse(manifestText), paths); + await writeFile(productsPath, `${JSON.stringify(rebuilt.products, null, 2)}\n`, 'utf8'); + if (manifestUpdate.appended.length > 0) { + await writeFile(manifestPath, `${JSON.stringify(manifestUpdate.manifest, null, 2)}\n`, 'utf8'); + } + return { + records: records.length, + unmatched: rebuilt.unmatched, + appended: manifestUpdate.appended, + missing: manifestUpdate.missing + }; +} + +function parseArgs(argv) { + const options = { + issuesDir: join(REPO_ROOT, 'web/data/issues'), + productsPath: join(REPO_ROOT, 'web/data/products.json'), + manifestPath: join(REPO_ROOT, 'web/data/product-tree-manifest.json') + }; + const flags = new Map([ + ['--issues-dir', 'issuesDir'], + ['--products-json', 'productsPath'], + ['--manifest-json', 'manifestPath'] + ]); + for (let index = 0; index < argv.length; index += 2) { + const property = flags.get(argv[index]); + if (!property || argv[index + 1] === undefined) throw new Error(`Invalid argument: ${argv[index] ?? ''}`); + options[property] = resolve(argv[index + 1]); + } + return options; +} + +export async function runUpdateProductsCli(argv = process.argv.slice(2)) { + const options = parseArgs(argv); + const result = await updateProductsFromIssues(options); + console.log(`Updated ${options.productsPath} from ${result.records} issue files.`); + console.log(`Appended ${result.appended.length} manifest path(s).`); + if (result.missing.length) console.warn(`${result.missing.length} manifest path(s) are no longer rendered; slots were preserved.`); + if (result.unmatched.length) console.warn(`${result.unmatched.length} issue file(s) did not match a products.json leaf.`); + return result; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + await runUpdateProductsCli(); +} diff --git a/scripts/update_products_from_issues.py b/scripts/update_products_from_issues.py deleted file mode 100644 index 754c286..0000000 --- a/scripts/update_products_from_issues.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""Update products.json entries from discovered issue data files.""" - -from __future__ import annotations - -import argparse -import json -import re -from pathlib import Path -from typing import Any - - -ISSUE_TYPES = {"addressed", "known"} -VERSION_FILENAME_RE = re.compile(r"^(?P[^.]+(?:\.[^.]+)*)\.md$") -HOTFIX_RE = re.compile(r"^(?P\d+(?:\.\d+)*)(?:-h(?P\d+))?$") - - -def parse_args() -> argparse.Namespace: - """Parse command line arguments for input and output paths.""" - - parser = argparse.ArgumentParser( - description="Update products.json from issue files." - ) - parser.add_argument( - "--issues-dir", - default="web/data/issues", - help="Directory containing product issue files.", - ) - parser.add_argument( - "--products-json", - default="web/data/products.json", - help="Path to products.json.", - ) - return parser.parse_args() - - -def collect_issue_files(issues_dir: Path) -> list[tuple[str, str, str, str]]: - """Collect all issue files with '.md' naming.""" - - records: list[tuple[str, str, str, str]] = [] - - for file_path in sorted(p for p in issues_dir.rglob("*") if p.is_file()): - relative = file_path.relative_to(issues_dir) - if len(relative.parts) != 3: - continue - - product, issue_type, filename = relative.parts - if issue_type not in ISSUE_TYPES: - continue - - version = parse_issue_filename(filename) - if version is None: - continue - - records.append((product, issue_type, version, filename)) - - return records - - -def parse_issue_filename(filename: str) -> str | None: - """Parse '.md' issue filenames and return version.""" - - match = VERSION_FILENAME_RE.match(filename) - if not match: - return None - - return match.group("version") - - -def clear_leaf_issue_arrays(node: Any) -> None: - """Reset addressed/known arrays on every issue leaf in products.json.""" - - if isinstance(node, dict): - if "addressed" in node or "known" in node: - node["addressed"] = [] - node["known"] = [] - return - - for value in node.values(): - clear_leaf_issue_arrays(value) - - -def is_issue_leaf(node: Any) -> bool: - """Return whether a node is a leaf containing issue arrays.""" - - return isinstance(node, dict) and ("addressed" in node or "known" in node) - - -def pick_target_leaf_path(products: dict[str, Any], product: str, version: str) -> list[str] | None: - """Choose the best matching product tree path for a version.""" - - parts = version.split(".") - if not parts: - return None - - major = parts[0] - minor = f"{parts[0]}.{parts[1]}" if len(parts) > 1 else parts[0] - - candidates = [ - [product, major, minor], - [product, major], - [product], - ] - - for path in candidates: - node = get_path(products, path) - if is_issue_leaf(node): - return path - - return None - - -def get_path(data: Any, path: list[str]) -> Any: - """Traverse nested dict keys and return the value at path, if present.""" - - current = data - for key in path: - if not isinstance(current, dict) or key not in current: - return None - current = current[key] - return current - - -def parse_release_prefix(prefix: str) -> tuple[tuple[int, ...], int]: - """Extract numeric base version parts and optional hotfix number.""" - - match = HOTFIX_RE.match(prefix) - if not match: - return tuple(), -1 - - base_parts = tuple(int(part) for part in match.group("base").split(".")) - hotfix_group = match.group("hotfix") - hotfix = int(hotfix_group) if hotfix_group is not None else -1 - return base_parts, hotfix - - -def filename_sort_key(filename: str) -> tuple[tuple[int, ...], int, int, str]: - """Build a sort key that orders base versions before hotfixes.""" - - prefix = filename.rsplit(".", 1)[0] - base_parts, hotfix = parse_release_prefix(prefix) - is_hotfix = 1 if hotfix >= 0 else 0 - return base_parts, is_hotfix, hotfix, filename - - -def add_file_to_leaf( - products: dict[str, Any], - path: list[str], - issue_type: str, - filename: str, -) -> None: - """Insert an issue filename into the target leaf with stable sorting.""" - - leaf = get_path(products, path) - if not isinstance(leaf, dict): - return - - files = leaf.get(issue_type) - if not isinstance(files, list): - files = [] - - files.append(filename) - leaf[issue_type] = sorted(set(files), key=filename_sort_key) - - -def update_products(issues_dir: Path, products_path: Path) -> None: - """Rebuild product issue file lists from crawled issue files.""" - - if not issues_dir.is_dir(): - raise FileNotFoundError(f"Issues directory not found: {issues_dir}") - if not products_path.is_file(): - raise FileNotFoundError(f"products.json not found: {products_path}") - - with products_path.open("r", encoding="utf-8") as f: - products = json.load(f) - - if not isinstance(products, dict): - raise ValueError("products.json must contain a top-level object") - - records = collect_issue_files(issues_dir) - if not records: - raise ValueError( - "No '.md' issue files found; refusing to rewrite products.json" - ) - - clear_leaf_issue_arrays(products) - - for product, issue_type, version, filename in sorted( - records, - key=lambda r: (r[0], r[1], r[2]), - ): - path = pick_target_leaf_path(products, product, version) - if path is None: - continue - add_file_to_leaf(products, path, issue_type, filename) - - with products_path.open("w", encoding="utf-8") as f: - json.dump(products, f, indent=2) - f.write("\n") - - -def main() -> int: - """Entrypoint for command-line usage.""" - - args = parse_args() - script_dir = Path(__file__).resolve().parent - repo_root = script_dir.parent - issues_dir = (repo_root / args.issues_dir).resolve() - products_path = (repo_root / args.products_json).resolve() - - update_products(issues_dir=issues_dir, products_path=products_path) - print(f"Updated {products_path} from {issues_dir}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test/fixtures/PAN-OS-12.1.2-addressed/expected-parsed.json b/test/fixtures/PAN-OS-12.1.2-addressed/expected-parsed.json new file mode 100644 index 0000000..40d5a17 --- /dev/null +++ b/test/fixtures/PAN-OS-12.1.2-addressed/expected-parsed.json @@ -0,0 +1,30 @@ +[ + { + "id": "PAN-242777", + "summary": "Fixed an issue where users previously reported limitations due to session count caps when utilizing **Web Proxy** features on PA-5400 Series Firewalls. To address these performance complaints and support higher traffic volumes, we have increased the maximum session capacity on specific **PA-5400F** series platforms, leveraging available system memory. This update ensures greater capacity and stability for high-volume environments.The supported session limits are:| Platform | Max Sessions |\n| -------- | ------------ |\n| PA-5410 | 95K |\n| PA-5420 | 95K |\n| PA-5430 | 95K |\n| PA-5440 | 225K |\n| PA-5445 | 250K |\n| PA-5450 | 1.28M |", + "resolved": "", + "caveat": "", + "renderedHtml": "

Fixed an issue where users previously reported limitations due to session count caps when utilizing Web Proxy features on PA-5400 Series Firewalls. To address these performance complaints and support higher traffic volumes, we have increased the maximum session capacity on specific PA-5400F series platforms, leveraging available system memory. This update ensures greater capacity and stability for high-volume environments.The supported session limits are:| Platform | Max Sessions |\n| -------- | ------------ |\n| PA-5410 | 95K |\n| PA-5420 | 95K |\n| PA-5430 | 95K |\n| PA-5440 | 225K |\n| PA-5445 | 250K |\n| PA-5450 | 1.28M |

\n" + }, + { + "id": "PAN-291499", + "summary": "Fixed an issue where newly deployed firewalls were unable to connect to the Strata Logging Service (SLS) until after a reboot, license fetch, or management server restart.", + "resolved": "", + "caveat": "VM-Series firewalls on Amazon Web Services (AWS) environments only", + "renderedHtml": "
Caveat: VM-Series firewalls on Amazon Web Services (AWS) environments only

Fixed an issue where newly deployed firewalls were unable to connect to the Strata Logging Service (SLS) until after a reboot, license fetch, or management server restart.

\n" + }, + { + "id": "PAN-288726", + "summary": "Fixed an issue where the useridd process stopped responding due to a Security policy rule ID being set to 0, which caused the last configuration retrieval to fail.", + "resolved": "", + "caveat": "", + "renderedHtml": "

Fixed an issue where the useridd process stopped responding due to a Security policy rule ID being set to 0, which caused the last configuration retrieval to fail.

\n" + }, + { + "id": "PAN-287133", + "summary": "Fixed an issue on the Panorama web interface where assigning a policy rule to a group at the top or bottom of the list changed the order of other policy rules.", + "resolved": "", + "caveat": "", + "renderedHtml": "

Fixed an issue on the Panorama web interface where assigning a policy rule to a group at the top or bottom of the list changed the order of other policy rules.

\n" + } +] diff --git a/test/update-products-from-issues.test.mjs b/test/update-products-from-issues.test.mjs new file mode 100644 index 0000000..02650bc --- /dev/null +++ b/test/update-products-from-issues.test.mjs @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import test from 'node:test'; +import { buildProductTree, enumerateTreePaths } from '../web/js/product-tree-model.js'; +import { updateProductsFromIssues } from '../scripts/update-products-from-issues.mjs'; + +test('product updater rebuilds products and append-only manifest idempotently', async t => { + const root = await mkdtemp(join(tmpdir(), 'firewallissues-update-')); + t.after(() => rm(root, { recursive: true, force: true })); + const issuesDir = join(root, 'issues'); + const productsPath = join(root, 'products.json'); + const manifestPath = join(root, 'manifest.json'); + await mkdir(join(issuesDir, 'Product', 'addressed'), { recursive: true }); + await mkdir(join(issuesDir, 'Product', 'known'), { recursive: true }); + await writeFile(join(issuesDir, 'Product', 'addressed', '1.2.3-h2.md'), 'addressed'); + await writeFile(join(issuesDir, 'Product', 'addressed', '1.2.3.md'), 'addressed'); + await writeFile(join(issuesDir, 'Product', 'known', '1.2.4.md'), 'known'); + + const initialProducts = { + Product: { + 1: { + '1.2': { + addressed: ['obsolete.md'], + known: [] + } + } + } + }; + const rootPath = JSON.stringify(['Product']); + await writeFile(productsPath, `${JSON.stringify(initialProducts, null, 2)}\n`); + await writeFile(manifestPath, `${JSON.stringify({ schema: 1, nodes: [rootPath, 'legacy-path', null] }, null, 2)}\n`); + + const first = await updateProductsFromIssues({ issuesDir, productsPath, manifestPath }); + assert.equal(first.records, 3); + assert.equal(first.unmatched.length, 0); + assert.ok(first.appended.length > 0); + assert.deepEqual(first.missing, ['legacy-path']); + + const products = JSON.parse(await readFile(productsPath, 'utf8')); + assert.deepEqual(products.Product[1]['1.2'].addressed, ['1.2.3.md', '1.2.3-h2.md']); + assert.deepEqual(products.Product[1]['1.2'].known, ['1.2.4.md']); + + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); + assert.deepEqual(manifest.nodes.slice(0, 3), [rootPath, 'legacy-path', null]); + const renderedPaths = enumerateTreePaths(buildProductTree(products)); + assert.ok(renderedPaths.every(path => manifest.nodes.includes(path))); + + const second = await updateProductsFromIssues({ issuesDir, productsPath, manifestPath }); + assert.deepEqual(second.appended, []); + assert.deepEqual(JSON.parse(await readFile(manifestPath, 'utf8')), manifest); +}); + +test('product updater refuses to rewrite when no issue files are found', async t => { + const root = await mkdtemp(join(tmpdir(), 'firewallissues-empty-update-')); + t.after(() => rm(root, { recursive: true, force: true })); + const issuesDir = join(root, 'issues'); + const productsPath = join(root, 'products.json'); + const manifestPath = join(root, 'manifest.json'); + await mkdir(issuesDir); + await writeFile(productsPath, '{}\n'); + await writeFile(manifestPath, '{"schema":1,"nodes":[]}\n'); + + await assert.rejects( + updateProductsFromIssues({ issuesDir, productsPath, manifestPath }), + /No '\.md' issue files found/ + ); + assert.equal(await readFile(productsPath, 'utf8'), '{}\n'); +}); diff --git a/web/js/product-tree-model.js b/web/js/product-tree-model.js new file mode 100644 index 0000000..e3d42ad --- /dev/null +++ b/web/js/product-tree-model.js @@ -0,0 +1,144 @@ +import { getSortedKeys } from './sort.js'; + +export function expandFilePrefixLevel(root) { + if (!root || typeof root !== 'object') { + return root; + } + return transformNode(root); +} + +function transformNode(node) { + if (!node || typeof node !== 'object' || Array.isArray(node)) { + return node; + } + + const hasIssueArrays = Object.prototype.hasOwnProperty.call(node, 'addressed') || + Object.prototype.hasOwnProperty.call(node, 'known'); + const childKeys = Object.keys(node).filter(key => key !== 'addressed' && key !== 'known'); + const transformedChildren = Object.fromEntries( + childKeys.map(key => [key, transformNode(node[key])]) + ); + + if (!hasIssueArrays) { + return transformedChildren; + } + + const addressed = Array.isArray(node.addressed) ? node.addressed : []; + const known = Array.isArray(node.known) ? node.known : []; + if (addressed.length + known.length === 0) { + return childKeys.length > 0 ? transformedChildren : { addressed: [], known: [] }; + } + + const groupedByPrefix = {}; + for (const [issueType, files] of [['addressed', addressed], ['known', known]]) { + for (const fileName of files) { + const prefix = getFilePrefix(fileName); + groupedByPrefix[prefix] ??= { addressed: [], known: [] }; + groupedByPrefix[prefix][issueType].push(fileName); + } + } + + const transformedIssueFiles = groupHotfixPrefixes(groupedByPrefix); + return childKeys.length === 0 + ? transformedIssueFiles + : mergeTreeNodes(transformedChildren, transformedIssueFiles); +} + +function groupHotfixPrefixes(groupedByPrefix) { + const byFamily = {}; + for (const prefix of Object.keys(groupedByPrefix)) { + const family = getReleaseFamily(prefix); + byFamily[family] ??= []; + byFamily[family].push(prefix); + } + + const result = {}; + for (const [family, members] of Object.entries(byFamily)) { + const hasHotfixes = members.length > 1 || members.some(member => member !== family); + if (!hasHotfixes && members[0] === family) { + result[family] = groupedByPrefix[family]; + } else { + result[family] = Object.fromEntries( + members.map(member => [member, groupedByPrefix[member]]) + ); + } + } + return result; +} + +function getReleaseFamily(prefix) { + return String(prefix || '').replace(/-h\d+$/i, ''); +} + +function getFilePrefix(fileName) { + const baseName = String(fileName || ''); + const underscoreIndex = baseName.indexOf('_'); + if (underscoreIndex <= 0) { + return baseName.replace(/\.json$/i, '') || 'unknown'; + } + return baseName.slice(0, underscoreIndex); +} + +function mergeTreeNodes(left, right) { + if (!isObjectNode(left)) return right; + if (!isObjectNode(right)) return left; + const merged = { ...left }; + + for (const [key, rightValue] of Object.entries(right)) { + const leftValue = merged[key]; + if (key === 'addressed' || key === 'known') { + const leftFiles = Array.isArray(leftValue) ? leftValue : []; + const rightFiles = Array.isArray(rightValue) ? rightValue : []; + merged[key] = Array.from(new Set([...leftFiles, ...rightFiles])); + } else if (isObjectNode(leftValue) && isObjectNode(rightValue)) { + merged[key] = mergeTreeNodes(leftValue, rightValue); + } else { + merged[key] = rightValue; + } + } + return merged; +} + +function isObjectNode(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +export function toTreeNodes(value, path = []) { + return getSortedKeys(value) + .filter(key => key !== 'addressed' && key !== 'known') + .map(key => { + const childValue = value[key]; + const childPath = [...path, key]; + const isObject = childValue && typeof childValue === 'object'; + const hasFiles = isObject && ('addressed' in childValue || 'known' in childValue); + const children = isObject ? toTreeNodes(childValue, childPath) : []; + return { + id: JSON.stringify(childPath), + label: key, + selectable: hasFiles || children.length > 0, + metadata: { + path: childPath, + addressed: hasFiles && Array.isArray(childValue.addressed) ? childValue.addressed : [], + known: hasFiles && Array.isArray(childValue.known) ? childValue.known : [] + }, + ...(children.length > 0 ? { children } : {}) + }; + }); +} + +export function enumerateTreePaths(nodes) { + const paths = []; + function visit(children, parentPath = '') { + for (const node of children) { + const path = parentPath ? `${parentPath}>${node.id}` : node.id; + paths.push(path); + visit(node.children ?? [], path); + } + } + visit(nodes); + return paths; +} + +export function buildProductTree(products) { + return toTreeNodes(expandFilePrefixLevel(products)); +} diff --git a/web/js/script.js b/web/js/script.js index 63fafab..a22180e 100644 --- a/web/js/script.js +++ b/web/js/script.js @@ -1,6 +1,7 @@ import { loadIssuesForCheckedPaths } from './issues.js'; import { getCheckedFileRefs, initializeTree, renderProductTree, restoreTreeState } from './tree.js'; import { applyIssueSearchFilter, getIssueTypeFilters, initializeUI } from './ui.js'; +import { expandFilePrefixLevel } from './product-tree-model.js'; document.addEventListener('DOMContentLoaded', () => { initializeUI({ onSelectionChange: refreshIssuesForCurrentSelection }); @@ -29,151 +30,3 @@ function loadProductTree() { }) .catch(error => console.error('Error loading products:', error)); } - -function expandFilePrefixLevel(root) { - if (!root || typeof root !== 'object') { - return root; - } - - return transformNode(root); -} - -function transformNode(node) { - if (!node || typeof node !== 'object' || Array.isArray(node)) { - return node; - } - - const hasIssueArrays = Object.prototype.hasOwnProperty.call(node, 'addressed') || - Object.prototype.hasOwnProperty.call(node, 'known'); - const childKeys = Object.keys(node).filter(key => key !== 'addressed' && key !== 'known'); - - const transformedChildren = {}; - childKeys.forEach(key => { - transformedChildren[key] = transformNode(node[key]); - }); - - if (!hasIssueArrays) { - return transformedChildren; - } - - const addressed = Array.isArray(node.addressed) ? node.addressed : []; - const known = Array.isArray(node.known) ? node.known : []; - const allFiles = [...addressed, ...known]; - - if (allFiles.length === 0) { - if (childKeys.length > 0) { - return transformedChildren; - } - - return { - addressed: [], - known: [] - }; - } - - const groupedByPrefix = {}; - - addressed.forEach(fileName => { - const prefix = getFilePrefix(fileName); - if (!groupedByPrefix[prefix]) { - groupedByPrefix[prefix] = { addressed: [], known: [] }; - } - groupedByPrefix[prefix].addressed.push(fileName); - }); - - known.forEach(fileName => { - const prefix = getFilePrefix(fileName); - if (!groupedByPrefix[prefix]) { - groupedByPrefix[prefix] = { addressed: [], known: [] }; - } - groupedByPrefix[prefix].known.push(fileName); - }); - - const transformedIssueFiles = groupHotfixPrefixes(groupedByPrefix); - if (childKeys.length === 0) { - return transformedIssueFiles; - } - - return mergeTreeNodes(transformedChildren, transformedIssueFiles); -} - -function groupHotfixPrefixes(groupedByPrefix) { - const byFamily = {}; - - Object.keys(groupedByPrefix).forEach(prefix => { - const family = getReleaseFamily(prefix); - if (!byFamily[family]) { - byFamily[family] = []; - } - byFamily[family].push(prefix); - }); - - const result = {}; - - Object.keys(byFamily).forEach(family => { - const members = byFamily[family]; - const hasHotfixes = members.length > 1 || members.some(member => member !== family); - - if (!hasHotfixes && members[0] === family) { - result[family] = groupedByPrefix[family]; - return; - } - - const branch = {}; - members.forEach(member => { - branch[member] = groupedByPrefix[member]; - }); - result[family] = branch; - }); - - return result; -} - -function getReleaseFamily(prefix) { - return String(prefix || '').replace(/-h\d+$/i, ''); -} - -function getFilePrefix(fileName) { - const baseName = String(fileName || ''); - const underscoreIndex = baseName.indexOf('_'); - if (underscoreIndex <= 0) { - return baseName.replace(/\.json$/i, '') || 'unknown'; - } - return baseName.slice(0, underscoreIndex); -} - -function mergeTreeNodes(left, right) { - if (!isObjectNode(left)) { - return right; - } - if (!isObjectNode(right)) { - return left; - } - - const merged = { ...left }; - - Object.keys(right).forEach(key => { - const leftValue = merged[key]; - const rightValue = right[key]; - - if (key === 'addressed' || key === 'known') { - const leftFiles = Array.isArray(leftValue) ? leftValue : []; - const rightFiles = Array.isArray(rightValue) ? rightValue : []; - merged[key] = Array.from(new Set([...leftFiles, ...rightFiles])); - return; - } - - if (isObjectNode(leftValue) && isObjectNode(rightValue)) { - merged[key] = mergeTreeNodes(leftValue, rightValue); - return; - } - - merged[key] = rightValue; - }); - - return merged; -} - -function isObjectNode(value) { - return Boolean(value) && typeof value === 'object' && !Array.isArray(value); -} diff --git a/web/js/tree.js b/web/js/tree.js index d40f361..9b8d0b8 100644 --- a/web/js/tree.js +++ b/web/js/tree.js @@ -1,5 +1,5 @@ import { CheckboxTree, loadManifest } from '../vendor/checkbox-tree/checkbox-tree.js'; -import { getSortedKeys } from './sort.js'; +import { toTreeNodes } from './product-tree-model.js'; const STORAGE_KEY = 'tree-state-v2'; let productTree = null; @@ -14,6 +14,7 @@ export function initializeTree(containerId, onSelectionChange) { productTree = new CheckboxTree(container, { storageKey: STORAGE_KEY, initiallyCollapsed: true, + stateEncoding: 'tiered', onSelectionChange: event => { onSelectionChange(event); pushTreeState(); @@ -82,30 +83,6 @@ function pushTreeState() { } } -function toTreeNodes(value, path = []) { - return getSortedKeys(value) - .filter(key => key !== 'addressed' && key !== 'known') - .map(key => { - const childValue = value[key]; - const childPath = [...path, key]; - const isObject = childValue && typeof childValue === 'object'; - const hasFiles = isObject && ('addressed' in childValue || 'known' in childValue); - const children = isObject ? toTreeNodes(childValue, childPath) : []; - - return { - id: JSON.stringify(childPath), - label: key, - selectable: hasFiles || children.length > 0, - metadata: { - path: childPath, - addressed: hasFiles && Array.isArray(childValue.addressed) ? childValue.addressed : [], - known: hasFiles && Array.isArray(childValue.known) ? childValue.known : [] - }, - ...(children.length > 0 ? { children } : {}) - }; - }); -} - function dedupeFileRefs(fileRefs) { const seen = new Set(); return fileRefs.filter(({ productKey, fileName }) => { diff --git a/web/vendor/checkbox-tree/README.md b/web/vendor/checkbox-tree/README.md index a14d6f3..b8a8b67 100644 --- a/web/vendor/checkbox-tree/README.md +++ b/web/vendor/checkbox-tree/README.md @@ -43,8 +43,8 @@ tree.setData([ ]); ``` -For applications without a bundler, serve or copy the two files in `src/` and -use relative URLs: +For applications without a bundler, serve or copy the `src/` directory and use +relative URLs. The main module imports its state-codec modules alongside it: ```html @@ -78,6 +78,9 @@ Options: - `onSelectionChange` receives `{ selectedIds, selectedNodes }`. - `manifest` optionally supplies a parsed manifest and automatically restores state from `window.location.hash` after `setData()`. +- `stateEncoding` selects `"adaptive"` (the default), `"dense"`, or `"tiered"`. + The website's configured value is the URL-format contract and is not embedded + in the fragment. ## Shareable URL state @@ -105,6 +108,15 @@ Use `validateManifestEvolution(previous, next)` in a build check to reject removed slots and tombstone reuse; path changes are reported for deliberate rename/move review. +With `stateEncoding: "tiered"`, `c1` stores root checked state and each deeper +`cN` field stores nodes whose checked state differs from their parent. Expansion +does not inherit: every `xN` stores the actual expanded branches at that depth, +with collapsed as the baseline. Each tier field uses the adaptive codec. The +`n` field records manifest coverage so nodes appended later still default to +false. A non-selectable parent supplies a false checked baseline. With +`stateEncoding: "dense"`, `c` and `x` contain trimmed +least-significant-bit-first bitsets. + Styling is namespaced under `.checkbox-tree`. Override the custom properties `--checkbox-tree-indent`, `--checkbox-tree-toggle-size`, and `--checkbox-tree-hover-color` in the consuming application. diff --git a/web/vendor/checkbox-tree/checkbox-tree.js b/web/vendor/checkbox-tree/checkbox-tree.js index 4974fc6..0dbe2f9 100644 --- a/web/vendor/checkbox-tree/checkbox-tree.js +++ b/web/vendor/checkbox-tree/checkbox-tree.js @@ -1,311 +1,44 @@ +// Dependency-free checkbox tree widget. State serialization lives in focused sibling modules. +import { + PATH_DELIMITER, + assertStateEncoding +} from './state-codec.js'; +import { + restoreStateFromLocation, + serializeStateToFragment, + validateManifest +} from './tree-state.js'; + +export { + PATH_DELIMITER, + SERIALIZATION_VERSION, + base64UrlToBytes, + bytesToBase64Url, + decodeAdaptiveBitset, + encodeAdaptiveBitset, + getBit, + loadManifest, + setBit +} from './state-codec.js'; +export { + applyTreeState, + decodeTreeState, + encodeTreeState, + restoreStateFromLocation, + serializeStateToFragment, + validateManifest, + validateManifestEvolution +} from './tree-state.js'; + const DEFAULT_OPTIONS = { initiallyCollapsed: true, initiallySelected: false, storageKey: null, onSelectionChange: null, - manifest: null + manifest: null, + stateEncoding: 'adaptive' }; -const SERIALIZATION_VERSION = 1; -const PATH_DELIMITER = '>'; - -export function loadManifest(source) { - let records; - let schema = 1; - - if (typeof source === 'string') { - records = source.split(/\r?\n/) - .map(line => line.trim()) - .filter(line => line && !line.startsWith('#')) - .map(line => line === '!deleted' ? null : line); - } else if (source && typeof source === 'object' && Array.isArray(source.nodes)) { - schema = source.schema; - records = source.nodes; - } else if (source && Array.isArray(source.slots)) { - return normalizeManifest(source.schema, source.slots.map(slot => slot?.deleted ? null : slot?.path)); - } else { - throw new TypeError('CheckboxTree manifest must be text or an object with a nodes array.'); - } - - return normalizeManifest(schema, records); -} - -function normalizeManifest(schema, records) { - if (schema !== 1) { - throw new Error(`Unsupported CheckboxTree manifest schema: ${schema}`); - } - - const pathToSlot = new Map(); - const slots = records.map((record, index) => { - if (record === null || record === '!deleted') { - return { path: null, deleted: true }; - } - if (typeof record !== 'string' || !record || record.trim() !== record) { - throw new TypeError(`Invalid CheckboxTree manifest entry at slot ${index}.`); - } - if (pathToSlot.has(record)) { - throw new Error(`Duplicate CheckboxTree manifest path: ${record}`); - } - pathToSlot.set(record, index); - return { path: record, deleted: false }; - }); - - return { schema: 1, slots, pathToSlot }; -} - -export function setBit(bytes, bitIndex, value) { - const byteIndex = Math.floor(bitIndex / 8); - const mask = 1 << (bitIndex % 8); - if (value) bytes[byteIndex] |= mask; - else bytes[byteIndex] &= ~mask; -} - -export function getBit(bytes, bitIndex) { - const byteIndex = Math.floor(bitIndex / 8); - return byteIndex < bytes.length && (bytes[byteIndex] & (1 << (bitIndex % 8))) !== 0; -} - -export function bytesToBase64Url(bytes) { - let binary = ''; - for (const byte of bytes) binary += String.fromCharCode(byte); - return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); -} - -export function base64UrlToBytes(value, maximumLength = Infinity) { - if (typeof value !== 'string' || !/^[A-Za-z0-9_-]*$/.test(value) || value.length % 4 === 1) { - throw new Error('Invalid Base64URL value.'); - } - if (Number.isFinite(maximumLength) && value.length > Math.ceil(maximumLength * 4 / 3) + 2) { - throw new Error('Encoded tree state exceeds the manifest size.'); - } - const base64 = value.replace(/-/g, '+').replace(/_/g, '/'); - const binary = atob(base64 + '='.repeat((4 - base64.length % 4) % 4)); - if (binary.length > maximumLength) throw new Error('Encoded tree state exceeds the manifest size.'); - return Uint8Array.from(binary, character => character.charCodeAt(0)); -} - -function trimTrailingZeroBytes(bytes) { - let end = bytes.length; - while (end && bytes[end - 1] === 0) end--; - return bytes.slice(0, end); -} - -function encodeUnsignedVarint(value) { - const bytes = []; - do { - let byte = value & 0x7f; - value = Math.floor(value / 128); - if (value) byte |= 0x80; - bytes.push(byte); - } while (value); - return bytes; -} - -function decodeUnsignedVarint(bytes, offset) { - let value = 0; - let multiplier = 1; - for (let index = offset; index < bytes.length && index < offset + 8; index++) { - const byte = bytes[index]; - value += (byte & 0x7f) * multiplier; - if ((byte & 0x80) === 0) return { value, offset: index + 1 }; - multiplier *= 128; - if (!Number.isSafeInteger(value) || !Number.isSafeInteger(multiplier)) break; - } - throw new Error('Invalid adaptive tree-state integer.'); -} - -function encodeSparseIndexes(indexes) { - const bytes = []; - let previous = -1; - for (const index of indexes) { - bytes.push(...encodeUnsignedVarint(index - previous - 1)); - previous = index; - } - return bytes; -} - -function decodeSparseIndexes(bytes, offset, limit) { - const indexes = new Set(); - let previous = -1; - while (offset < bytes.length) { - const decoded = decodeUnsignedVarint(bytes, offset); - const index = previous + decoded.value + 1; - if (index >= limit) throw new Error('Sparse tree state contains an out-of-range slot.'); - indexes.add(index); - previous = index; - offset = decoded.offset; - } - return indexes; -} - -export function encodeAdaptiveBitset(values, slotCount) { - const enabled = []; - const disabled = []; - const denseBytes = new Uint8Array(Math.ceil(slotCount / 8)); - for (const [slot, value] of values) { - (value ? enabled : disabled).push(slot); - setBit(denseBytes, slot, value); - } - enabled.sort((left, right) => left - right); - disabled.sort((left, right) => left - right); - if (enabled.length === 0) return new Uint8Array(); - - const candidates = [ - Uint8Array.from([0, ...trimTrailingZeroBytes(denseBytes)]), - Uint8Array.from([1, ...encodeSparseIndexes(enabled)]), - Uint8Array.from([2, ...encodeUnsignedVarint(slotCount), ...encodeSparseIndexes(disabled)]) - ]; - return candidates.reduce((shortest, candidate) => - candidate.length < shortest.length ? candidate : shortest - ); -} - -export function decodeAdaptiveBitset(bytes, slotCount) { - if (bytes.length === 0) return () => false; - const mode = bytes[0]; - if (mode === 0) { - const dense = bytes.slice(1); - if (dense.length > Math.ceil(slotCount / 8)) { - throw new Error('Dense tree state exceeds the manifest size.'); - } - return slot => getBit(dense, slot); - } - if (mode === 1) { - const enabled = decodeSparseIndexes(bytes, 1, slotCount); - return slot => enabled.has(slot); - } - if (mode === 2) { - const coverage = decodeUnsignedVarint(bytes, 1); - if (coverage.value > slotCount) throw new Error('Tree-state coverage exceeds the manifest size.'); - const disabled = decodeSparseIndexes(bytes, coverage.offset, coverage.value); - return slot => slot < coverage.value && !disabled.has(slot); - } - throw new Error('Unknown adaptive tree-state mode.'); -} - -function asManifest(manifest) { - return manifest?.pathToSlot instanceof Map ? manifest : loadManifest(manifest); -} - -export function validateManifest(manifest, tree) { - const normalized = asManifest(manifest); - const treePaths = new Set(tree.nodesByPath.keys()); - return { - valid: Array.from(treePaths).every(path => normalized.pathToSlot.has(path)), - unregisteredPaths: Array.from(treePaths).filter(path => !normalized.pathToSlot.has(path)), - missingPaths: normalized.slots - .filter(slot => !slot.deleted && !treePaths.has(slot.path)) - .map(slot => slot.path) - }; -} - -export function validateManifestEvolution(previous, next) { - const oldManifest = asManifest(previous); - const newManifest = asManifest(next); - const errors = []; - - if (newManifest.slots.length < oldManifest.slots.length) { - errors.push('Manifest slots cannot be removed; use tombstones.'); - } - oldManifest.slots.forEach((oldSlot, index) => { - const newSlot = newManifest.slots[index]; - if (!newSlot) return; - if (oldSlot.deleted && !newSlot.deleted) { - errors.push(`Tombstoned slot ${index} cannot be reused.`); - } - if (!oldSlot.deleted && !newSlot.deleted && oldSlot.path !== newSlot.path) { - errors.push(`Slot ${index} changed path; confirm this is a logical rename or move.`); - } - }); - - return { valid: errors.length === 0, errors }; -} - -export function encodeTreeState(tree, manifest) { - const normalized = asManifest(manifest); - const checkedValues = new Map(); - const expandedValues = new Map(); - - for (const [path, node] of tree.nodesByPath) { - const slot = normalized.pathToSlot.get(path); - if (slot === undefined) continue; - const item = tree.itemForNode(node.id); - const checkbox = item && tree.directCheckbox(item); - const toggle = item?.querySelector(':scope > .checkbox-tree__row > .checkbox-tree__toggle'); - checkedValues.set(slot, Boolean(checkbox?.checked)); - expandedValues.set(slot, toggle?.getAttribute('aria-expanded') === 'true'); - } - - const checkedValue = bytesToBase64Url(encodeAdaptiveBitset(checkedValues, normalized.slots.length)); - const expandedValue = bytesToBase64Url(encodeAdaptiveBitset(expandedValues, normalized.slots.length)); - return { version: SERIALIZATION_VERSION, checked: checkedValue || null, expanded: expandedValue || null }; -} - -export function decodeTreeState(fragment, manifest) { - const normalized = asManifest(manifest); - const value = fragment.startsWith('#') ? fragment.slice(1) : fragment; - const params = new URLSearchParams(value); - if (params.get('v') !== String(SERIALIZATION_VERSION)) { - return { applied: false, version: params.get('v'), warnings: ['Missing or unsupported serialization version.'] }; - } - const maximumLength = Math.max( - Math.ceil(normalized.slots.length / 8) + 1, - normalized.slots.length * 2 + 10 - ); - try { - const checked = base64UrlToBytes(params.get('c') ?? '', maximumLength); - const expanded = base64UrlToBytes(params.get('x') ?? '', maximumLength); - decodeAdaptiveBitset(checked, normalized.slots.length); - decodeAdaptiveBitset(expanded, normalized.slots.length); - return { - applied: true, - version: SERIALIZATION_VERSION, - checked, - expanded, - warnings: [] - }; - } catch (error) { - return { applied: false, version: SERIALIZATION_VERSION, warnings: [error.message] }; - } -} - -export function applyTreeState(tree, manifest, state) { - if (!state?.applied) return false; - const normalized = asManifest(manifest); - let isChecked; - let isExpanded; - try { - isChecked = decodeAdaptiveBitset(state.checked, normalized.slots.length); - isExpanded = decodeAdaptiveBitset(state.expanded, normalized.slots.length); - } catch { - return false; - } - for (const [path, node] of tree.nodesByPath) { - const slot = normalized.pathToSlot.get(path); - if (slot === undefined) continue; - const item = tree.itemForNode(node.id); - const checkbox = item && tree.directCheckbox(item); - if (checkbox && !checkbox.disabled) checkbox.checked = isChecked(slot); - tree.setExpanded(item, isExpanded(slot)); - } - tree.updateAllParentCheckboxes(); - return true; -} - -export function serializeStateToFragment(tree, manifest) { - const state = encodeTreeState(tree, manifest); - const params = new URLSearchParams({ v: String(state.version) }); - if (state.checked) params.set('c', state.checked); - if (state.expanded) params.set('x', state.expanded); - return `#${params}`; -} - -export function restoreStateFromLocation(tree, manifest, location = window.location) { - const state = decodeTreeState(location.hash ?? '', manifest); - if (state.applied) applyTreeState(tree, manifest, state); - return state; -} - export class CheckboxTree { constructor(container, options = {}) { if (!(container instanceof Element)) { @@ -314,8 +47,10 @@ export class CheckboxTree { this.container = container; this.options = { ...DEFAULT_OPTIONS, ...options }; + assertStateEncoding(this.options.stateEncoding); this.nodesById = new Map(); this.nodesByPath = new Map(); + this.itemsById = new Map(); this.rootNodes = []; this.handleChange = this.handleChange.bind(this); this.handleClick = this.handleClick.bind(this); @@ -332,6 +67,7 @@ export class CheckboxTree { this.nodesById.clear(); this.nodesByPath.clear(); + this.itemsById.clear(); this.rootNodes = nodes; this.indexNodes(nodes); this.render(); @@ -371,6 +107,7 @@ export class CheckboxTree { this.container.replaceChildren(); this.nodesById.clear(); this.nodesByPath.clear(); + this.itemsById.clear(); } indexNodes(nodes, parentPath = '') { @@ -408,6 +145,7 @@ export class CheckboxTree { const item = document.createElement('div'); item.className = 'checkbox-tree__item'; item.dataset.nodeId = node.id; + this.itemsById.set(node.id, item); const row = document.createElement('div'); row.className = 'checkbox-tree__row'; @@ -524,8 +262,7 @@ export class CheckboxTree { } itemForNode(id) { - return Array.from(this.container.querySelectorAll('.checkbox-tree__item')) - .find(item => item.dataset.nodeId === id) ?? null; + return this.itemsById.get(id) ?? null; } updateAncestorCheckboxes(item) { diff --git a/web/vendor/checkbox-tree/state-codec.js b/web/vendor/checkbox-tree/state-codec.js new file mode 100644 index 0000000..8a982a8 --- /dev/null +++ b/web/vendor/checkbox-tree/state-codec.js @@ -0,0 +1,206 @@ +// Manifest parsing and binary codecs. This module has no DOM dependencies. +export const SERIALIZATION_VERSION = 1; +export const PATH_DELIMITER = '>'; +const STATE_ENCODINGS = new Set(['adaptive', 'dense', 'tiered']); + +export function loadManifest(source) { + let records; + let schema = 1; + + if (typeof source === 'string') { + records = source.split(/\r?\n/) + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')) + .map(line => line === '!deleted' ? null : line); + } else if (source && typeof source === 'object' && Array.isArray(source.nodes)) { + schema = source.schema; + records = source.nodes; + } else if (source && Array.isArray(source.slots)) { + return normalizeManifest(source.schema, source.slots.map(slot => slot?.deleted ? null : slot?.path)); + } else { + throw new TypeError('CheckboxTree manifest must be text or an object with a nodes array.'); + } + + return normalizeManifest(schema, records); +} + +function normalizeManifest(schema, records) { + if (schema !== 1) { + throw new Error(`Unsupported CheckboxTree manifest schema: ${schema}`); + } + + const pathToSlot = new Map(); + const slots = records.map((record, index) => { + if (record === null || record === '!deleted') { + return { path: null, deleted: true }; + } + if (typeof record !== 'string' || !record || record.trim() !== record) { + throw new TypeError(`Invalid CheckboxTree manifest entry at slot ${index}.`); + } + if (pathToSlot.has(record)) { + throw new Error(`Duplicate CheckboxTree manifest path: ${record}`); + } + pathToSlot.set(record, index); + return { path: record, deleted: false }; + }); + + return { schema: 1, slots, pathToSlot }; +} + +export function setBit(bytes, bitIndex, value) { + const byteIndex = Math.floor(bitIndex / 8); + const mask = 1 << (bitIndex % 8); + if (value) bytes[byteIndex] |= mask; + else bytes[byteIndex] &= ~mask; +} + +export function getBit(bytes, bitIndex) { + const byteIndex = Math.floor(bitIndex / 8); + return byteIndex < bytes.length && (bytes[byteIndex] & (1 << (bitIndex % 8))) !== 0; +} + +export function bytesToBase64Url(bytes) { + let binary = ''; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); +} + +export function base64UrlToBytes(value, maximumLength = Infinity) { + if (typeof value !== 'string' || !/^[A-Za-z0-9_-]*$/.test(value) || value.length % 4 === 1) { + throw new Error('Invalid Base64URL value.'); + } + if (Number.isFinite(maximumLength) && value.length > Math.ceil(maximumLength * 4 / 3) + 2) { + throw new Error('Encoded tree state exceeds the manifest size.'); + } + const base64 = value.replace(/-/g, '+').replace(/_/g, '/'); + const binary = atob(base64 + '='.repeat((4 - base64.length % 4) % 4)); + if (binary.length > maximumLength) throw new Error('Encoded tree state exceeds the manifest size.'); + return Uint8Array.from(binary, character => character.charCodeAt(0)); +} + +function trimTrailingZeroBytes(bytes) { + let end = bytes.length; + while (end && bytes[end - 1] === 0) end--; + return bytes.slice(0, end); +} + +function encodeUnsignedVarint(value) { + const bytes = []; + do { + let byte = value & 0x7f; + value = Math.floor(value / 128); + if (value) byte |= 0x80; + bytes.push(byte); + } while (value); + return bytes; +} + +function decodeUnsignedVarint(bytes, offset) { + let value = 0; + let multiplier = 1; + for (let index = offset; index < bytes.length && index < offset + 8; index++) { + const byte = bytes[index]; + value += (byte & 0x7f) * multiplier; + if ((byte & 0x80) === 0) return { value, offset: index + 1 }; + multiplier *= 128; + if (!Number.isSafeInteger(value) || !Number.isSafeInteger(multiplier)) break; + } + throw new Error('Invalid adaptive tree-state integer.'); +} + +function encodeSparseIndexes(indexes) { + const bytes = []; + let previous = -1; + for (const index of indexes) { + bytes.push(...encodeUnsignedVarint(index - previous - 1)); + previous = index; + } + return bytes; +} + +function decodeSparseIndexes(bytes, offset, limit) { + const indexes = new Set(); + let previous = -1; + while (offset < bytes.length) { + const decoded = decodeUnsignedVarint(bytes, offset); + const index = previous + decoded.value + 1; + if (index >= limit) throw new Error('Sparse tree state contains an out-of-range slot.'); + indexes.add(index); + previous = index; + offset = decoded.offset; + } + return indexes; +} + +export function encodeAdaptiveBitset(values, slotCount) { + const enabled = []; + const disabled = []; + const denseBytes = new Uint8Array(Math.ceil(slotCount / 8)); + for (const [slot, value] of values) { + (value ? enabled : disabled).push(slot); + setBit(denseBytes, slot, value); + } + enabled.sort((left, right) => left - right); + disabled.sort((left, right) => left - right); + if (enabled.length === 0) return new Uint8Array(); + + const candidates = [ + Uint8Array.from([0, ...trimTrailingZeroBytes(denseBytes)]), + Uint8Array.from([1, ...encodeSparseIndexes(enabled)]), + Uint8Array.from([2, ...encodeUnsignedVarint(slotCount), ...encodeSparseIndexes(disabled)]) + ]; + return candidates.reduce((shortest, candidate) => + candidate.length < shortest.length ? candidate : shortest + ); +} + +export function decodeAdaptiveBitset(bytes, slotCount) { + if (bytes.length === 0) return () => false; + const mode = bytes[0]; + if (mode === 0) { + const dense = bytes.slice(1); + if (dense.length > Math.ceil(slotCount / 8)) { + throw new Error('Dense tree state exceeds the manifest size.'); + } + return slot => getBit(dense, slot); + } + if (mode === 1) { + const enabled = decodeSparseIndexes(bytes, 1, slotCount); + return slot => enabled.has(slot); + } + if (mode === 2) { + const coverage = decodeUnsignedVarint(bytes, 1); + if (coverage.value > slotCount) throw new Error('Tree-state coverage exceeds the manifest size.'); + const disabled = decodeSparseIndexes(bytes, coverage.offset, coverage.value); + return slot => slot < coverage.value && !disabled.has(slot); + } + throw new Error('Unknown adaptive tree-state mode.'); +} + + +export function encodeDenseBitset(values, slotCount) { + const bytes = new Uint8Array(Math.ceil(slotCount / 8)); + for (const [slot, value] of values) setBit(bytes, slot, value); + return trimTrailingZeroBytes(bytes); +} + +export function decodeDenseBitset(bytes, slotCount) { + if (bytes.length > Math.ceil(slotCount / 8)) { + throw new Error('Dense tree state exceeds the manifest size.'); + } + return slot => getBit(bytes, slot); +} + +export function maximumEncodedLength(slotCount) { + return Math.max(Math.ceil(slotCount / 8) + 1, slotCount * 2 + 10); +} + +export function assertStateEncoding(encoding) { + if (!STATE_ENCODINGS.has(encoding)) { + throw new Error(`Unsupported CheckboxTree state encoding: ${encoding}`); + } +} + +export function asManifestSource(manifest) { + return manifest?.pathToSlot instanceof Map ? manifest : loadManifest(manifest); +} diff --git a/web/vendor/checkbox-tree/tree-state.js b/web/vendor/checkbox-tree/tree-state.js new file mode 100644 index 0000000..ac6ed6a --- /dev/null +++ b/web/vendor/checkbox-tree/tree-state.js @@ -0,0 +1,310 @@ +// Adapts pure state codecs to CheckboxTree's rendered node model and URL format. +import { + PATH_DELIMITER, + SERIALIZATION_VERSION, + asManifestSource as asManifest, + assertStateEncoding, + base64UrlToBytes, + bytesToBase64Url, + decodeAdaptiveBitset, + decodeDenseBitset, + encodeAdaptiveBitset, + encodeDenseBitset, + maximumEncodedLength +} from './state-codec.js'; + +export function validateManifest(manifest, tree) { + const normalized = asManifest(manifest); + const treePaths = new Set(tree.nodesByPath.keys()); + return { + valid: Array.from(treePaths).every(path => normalized.pathToSlot.has(path)), + unregisteredPaths: Array.from(treePaths).filter(path => !normalized.pathToSlot.has(path)), + missingPaths: normalized.slots + .filter(slot => !slot.deleted && !treePaths.has(slot.path)) + .map(slot => slot.path) + }; +} + +export function validateManifestEvolution(previous, next) { + const oldManifest = asManifest(previous); + const newManifest = asManifest(next); + const errors = []; + + if (newManifest.slots.length < oldManifest.slots.length) { + errors.push('Manifest slots cannot be removed; use tombstones.'); + } + oldManifest.slots.forEach((oldSlot, index) => { + const newSlot = newManifest.slots[index]; + if (!newSlot) return; + if (oldSlot.deleted && !newSlot.deleted) { + errors.push(`Tombstoned slot ${index} cannot be reused.`); + } + if (!oldSlot.deleted && !newSlot.deleted && oldSlot.path !== newSlot.path) { + errors.push(`Slot ${index} changed path; confirm this is a logical rename or move.`); + } + }); + + return { valid: errors.length === 0, errors }; +} + +function collectTreeValues(tree, manifest) { + const normalized = asManifest(manifest); + const checkedValues = new Map(); + const expandedValues = new Map(); + + for (const [path, node] of tree.nodesByPath) { + const slot = normalized.pathToSlot.get(path); + if (slot === undefined) continue; + const item = tree.itemForNode(node.id); + const checkbox = item && tree.directCheckbox(item); + const toggle = item?.querySelector(':scope > .checkbox-tree__row > .checkbox-tree__toggle'); + checkedValues.set(slot, Boolean(checkbox?.checked)); + expandedValues.set(slot, toggle?.getAttribute('aria-expanded') === 'true'); + } + + return { normalized, checkedValues, expandedValues }; +} + +function nodeState(tree, node) { + const item = tree.itemForNode(node.id); + const checkbox = item && tree.directCheckbox(item); + const toggle = item?.querySelector(':scope > .checkbox-tree__row > .checkbox-tree__toggle'); + return { + checked: Boolean(checkbox?.checked), + hasChecked: Boolean(checkbox), + expanded: toggle?.getAttribute('aria-expanded') === 'true', + hasExpanded: Boolean(toggle) + }; +} + +function encodeTieredState(tree, normalized) { + const checkedByDepth = new Map(); + const expandedByDepth = new Map(); + const statesByPath = new Map(); + + for (const [path, node] of tree.nodesByPath) { + const slot = normalized.pathToSlot.get(path); + if (slot === undefined) continue; + const depth = path.split(PATH_DELIMITER).length; + const parentPath = path.includes(PATH_DELIMITER) + ? path.slice(0, path.lastIndexOf(PATH_DELIMITER)) + : null; + const parent = statesByPath.get(parentPath) ?? { + checked: false, + hasChecked: false, + expanded: false, + hasExpanded: false + }; + const current = nodeState(tree, node); + if (current.hasChecked) { + checkedByDepth.set(depth, checkedByDepth.get(depth) ?? new Map()); + checkedByDepth.get(depth).set(slot, current.checked !== (parent.hasChecked ? parent.checked : false)); + } + if (current.hasExpanded) { + expandedByDepth.set(depth, expandedByDepth.get(depth) ?? new Map()); + expandedByDepth.get(depth).set(slot, current.expanded); + } + statesByPath.set(path, current); + } + + const fields = new Map([['n', String(normalized.slots.length)]]); + for (const [depth, values] of checkedByDepth) { + const encoded = bytesToBase64Url(encodeAdaptiveBitset(values, normalized.slots.length)); + if (encoded) fields.set(`c${depth}`, encoded); + } + for (const [depth, values] of expandedByDepth) { + const encoded = bytesToBase64Url(encodeAdaptiveBitset(values, normalized.slots.length)); + if (encoded) fields.set(`x${depth}`, encoded); + } + return fields; +} + +export function encodeTreeState(tree, manifest, encoding = tree.options?.stateEncoding ?? 'adaptive') { + assertStateEncoding(encoding); + const { normalized, checkedValues, expandedValues } = collectTreeValues(tree, manifest); + + if (encoding === 'tiered') { + return { version: SERIALIZATION_VERSION, encoding, fields: encodeTieredState(tree, normalized) }; + } + + const encode = encoding === 'dense' ? encodeDenseBitset : encodeAdaptiveBitset; + const checkedValue = bytesToBase64Url(encode(checkedValues, normalized.slots.length)); + const expandedValue = bytesToBase64Url(encode(expandedValues, normalized.slots.length)); + return { + version: SERIALIZATION_VERSION, + encoding, + checked: checkedValue || null, + expanded: expandedValue || null + }; +} + +function decodeField(params, name, normalized, encoding) { + const maximumLength = encoding === 'dense' + ? Math.ceil(normalized.slots.length / 8) + : maximumEncodedLength(normalized.slots.length); + const bytes = base64UrlToBytes(params.get(name) ?? '', maximumLength); + const decode = encoding === 'dense' ? decodeDenseBitset : decodeAdaptiveBitset; + decode(bytes, normalized.slots.length); + return bytes; +} + +export function decodeTreeState(fragment, manifest, encoding = 'adaptive') { + assertStateEncoding(encoding); + const normalized = asManifest(manifest); + const value = fragment.startsWith('#') ? fragment.slice(1) : fragment; + const params = new URLSearchParams(value); + if (params.get('v') !== String(SERIALIZATION_VERSION)) { + return { applied: false, version: params.get('v'), encoding, warnings: ['Missing or unsupported serialization version.'] }; + } + try { + if (encoding === 'tiered') { + if (params.has('c') || params.has('x')) { + throw new Error('Tiered tree state contains non-tiered fields.'); + } + const maximumDepth = normalized.slots.reduce((depth, slot) => + slot.deleted ? depth : Math.max(depth, slot.path.split(PATH_DELIMITER).length) + , 0); + for (const key of params.keys()) { + const match = /^([cx])(\d+)$/.exec(key); + if (match && (Number(match[2]) < 1 || Number(match[2]) > maximumDepth)) { + throw new Error(`Tree state contains an invalid tier: ${key}`); + } + } + const coverage = Number(params.get('n')); + if (!Number.isSafeInteger(coverage) || coverage < 0 || coverage > normalized.slots.length) { + throw new Error('Tiered tree state has invalid manifest coverage.'); + } + const checkedTiers = new Map(); + const expandedTiers = new Map(); + for (let depth = 1; depth <= maximumDepth; depth++) { + checkedTiers.set(depth, decodeField(params, `c${depth}`, normalized, 'adaptive')); + expandedTiers.set(depth, decodeField(params, `x${depth}`, normalized, 'adaptive')); + } + return { + applied: true, + version: SERIALIZATION_VERSION, + encoding, + coverage, + checkedTiers, + expandedTiers, + warnings: [] + }; + } + + for (const key of params.keys()) { + if (key === 'n' || /^[cx]\d+$/.test(key)) { + throw new Error('Tree state contains tiered fields for a non-tiered encoding.'); + } + } + + const checked = decodeField(params, 'c', normalized, encoding); + const expanded = decodeField(params, 'x', normalized, encoding); + return { + applied: true, + version: SERIALIZATION_VERSION, + encoding, + checked, + expanded, + warnings: [] + }; + } catch (error) { + return { applied: false, version: SERIALIZATION_VERSION, encoding, warnings: [error.message] }; + } +} + +export function applyTreeState(tree, manifest, state) { + if (!state?.applied) return false; + const normalized = asManifest(manifest); + if (state.encoding === 'tiered') return applyTieredTreeState(tree, normalized, state); + let isChecked; + let isExpanded; + try { + const decode = state.encoding === 'dense' ? decodeDenseBitset : decodeAdaptiveBitset; + isChecked = decode(state.checked, normalized.slots.length); + isExpanded = decode(state.expanded, normalized.slots.length); + } catch { + return false; + } + for (const [path, node] of tree.nodesByPath) { + const slot = normalized.pathToSlot.get(path); + if (slot === undefined) continue; + const item = tree.itemForNode(node.id); + const checkbox = item && tree.directCheckbox(item); + if (checkbox && !checkbox.disabled) checkbox.checked = isChecked(slot); + tree.setExpanded(item, isExpanded(slot)); + } + tree.updateAllParentCheckboxes(); + return true; +} + +function applyTieredTreeState(tree, normalized, state) { + const resolvedByPath = new Map(); + const checkedDecoders = new Map(); + const expandedDecoders = new Map(); + try { + for (const [depth, bytes] of state.checkedTiers) { + checkedDecoders.set(depth, decodeAdaptiveBitset(bytes, normalized.slots.length)); + } + for (const [depth, bytes] of state.expandedTiers) { + expandedDecoders.set(depth, decodeAdaptiveBitset(bytes, normalized.slots.length)); + } + } catch { + return false; + } + + for (const [path, node] of tree.nodesByPath) { + const slot = normalized.pathToSlot.get(path); + if (slot === undefined) continue; + const depth = path.split(PATH_DELIMITER).length; + const parentPath = path.includes(PATH_DELIMITER) + ? path.slice(0, path.lastIndexOf(PATH_DELIMITER)) + : null; + const parent = resolvedByPath.get(parentPath) ?? { + checked: false, + hasChecked: false, + expanded: false, + hasExpanded: false + }; + const item = tree.itemForNode(node.id); + const checkbox = item && tree.directCheckbox(item); + const toggle = item?.querySelector(':scope > .checkbox-tree__row > .checkbox-tree__toggle'); + const isCovered = slot < state.coverage; + const checked = checkbox && isCovered + ? (parent.hasChecked ? parent.checked : false) !== checkedDecoders.get(depth)(slot) + : false; + const expanded = Boolean(toggle && isCovered && expandedDecoders.get(depth)(slot)); + if (checkbox && !checkbox.disabled) checkbox.checked = checked; + tree.setExpanded(item, expanded); + resolvedByPath.set(path, { + checked, + hasChecked: Boolean(checkbox), + expanded, + hasExpanded: Boolean(toggle) + }); + } + tree.updateAllParentCheckboxes(); + return true; +} + +export function serializeStateToFragment(tree, manifest, encoding = tree.options?.stateEncoding ?? 'adaptive') { + const state = encodeTreeState(tree, manifest, encoding); + const params = new URLSearchParams({ v: String(state.version) }); + if (state.encoding === 'tiered') { + for (const [name, value] of state.fields) params.set(name, value); + } else { + if (state.checked) params.set('c', state.checked); + if (state.expanded) params.set('x', state.expanded); + } + return `#${params}`; +} + +export function restoreStateFromLocation( + tree, + manifest, + location = window.location, + encoding = tree.options?.stateEncoding ?? 'adaptive' +) { + const state = decodeTreeState(location.hash ?? '', manifest, encoding); + if (state.applied) applyTreeState(tree, manifest, state); + return state; +}