Alter URL encoding

README.md

@@ -94,9 +94,11 @@ must never be reordered. Append new paths, replace removed paths with `null`, an
 edit a path in place for a logical rename or move. Node IDs therefore cannot
 contain `>`.
 
-The fragment contains independently trimmed checked (`c`) and expanded (`x`)
-bitsets. Bits are least-significant-bit first within each byte. Missing bits
-default to unchecked and collapsed; malformed or unsupported state is ignored.
+The fragment adaptively encodes checked (`c`) and expanded (`x`) state using the
+shortest of a dense bitset, sparse enabled slots, or sparse disabled slots.
+Sparse slot numbers are delta-encoded variable-length integers. A coverage value
+in the disabled-slot form ensures nodes appended later still default to unchecked
+and collapsed. Malformed or unsupported state is ignored.
 The helpers `encodeTreeState`, `decodeTreeState`, `applyTreeState`, and
 `restoreStateFromLocation` are also exported for custom integrations.
 Use `validateManifestEvolution(previous, next)` in a build check to reject

docs/tree-state-serialization.md

@@ -1,5 +1,29 @@
 # Tree UI State Serialization Design
 
+## Implementation amendment: adaptive fields
+
+The implemented version 1 format retains the manifest, URL parameters, stable
+slots, and default-state behavior described below, but adaptively encodes each
+`c` and `x` field. This amendment supersedes the document's later recommendation
+that fields always use dense bitsets.
+
+Each decoded field still represents one Boolean per manifest slot. Before
+Base64URL encoding, its first byte selects the shortest representation:
+
+* `0`: dense, least-significant-bit-first bitset
+* `1`: delta-encoded variable-length indexes of enabled slots
+* `2`: manifest coverage followed by delta-encoded indexes of disabled slots
+
+Mode 2's coverage value preserves forward compatibility: slots appended beyond
+the encoded coverage remain false rather than inheriting the enabled state.
+Empty all-false fields remain omitted. The encoder compares all three valid
+representations independently for checked and expanded state and emits the
+shortest. Sparse indexes are ascending, delta encoded, and stored as unsigned
+base-128 variable-length integers.
+
+This adaptive layout was adopted before deployment, so it uses serialization
+version `1`; no dense-only version 1 URLs need backward compatibility.
+
 ## 1. Purpose
 
 Implement a compact, shareable, client-side serialization format for storing tree widget state in a URL.

src/checkbox-tree.js

@@ -75,6 +75,9 @@ export function base64UrlToBytes(value, maximumLength = Infinity) {
     if (typeof value !== 'string' || !/^[A-Za-z0-9_-]*$/.test(value) || value.length % 4 === 1) {
         throw new Error('Invalid Base64URL value.');
     }
+    if (Number.isFinite(maximumLength) && value.length > Math.ceil(maximumLength * 4 / 3) + 2) {
+        throw new Error('Encoded tree state exceeds the manifest size.');
+    }
     const base64 = value.replace(/-/g, '+').replace(/_/g, '/');
     const binary = atob(base64 + '='.repeat((4 - base64.length % 4) % 4));
     if (binary.length > maximumLength) throw new Error('Encoded tree state exceeds the manifest size.');
@@ -87,6 +90,99 @@ function trimTrailingZeroBytes(bytes) {
     return bytes.slice(0, end);
 }
 
+function encodeUnsignedVarint(value) {
+    const bytes = [];
+    do {
+        let byte = value & 0x7f;
+        value = Math.floor(value / 128);
+        if (value) byte |= 0x80;
+        bytes.push(byte);
+    } while (value);
+    return bytes;
+}
+
+function decodeUnsignedVarint(bytes, offset) {
+    let value = 0;
+    let multiplier = 1;
+    for (let index = offset; index < bytes.length && index < offset + 8; index++) {
+        const byte = bytes[index];
+        value += (byte & 0x7f) * multiplier;
+        if ((byte & 0x80) === 0) return { value, offset: index + 1 };
+        multiplier *= 128;
+        if (!Number.isSafeInteger(value) || !Number.isSafeInteger(multiplier)) break;
+    }
+    throw new Error('Invalid adaptive tree-state integer.');
+}
+
+function encodeSparseIndexes(indexes) {
+    const bytes = [];
+    let previous = -1;
+    for (const index of indexes) {
+        bytes.push(...encodeUnsignedVarint(index - previous - 1));
+        previous = index;
+    }
+    return bytes;
+}
+
+function decodeSparseIndexes(bytes, offset, limit) {
+    const indexes = new Set();
+    let previous = -1;
+    while (offset < bytes.length) {
+        const decoded = decodeUnsignedVarint(bytes, offset);
+        const index = previous + decoded.value + 1;
+        if (index >= limit) throw new Error('Sparse tree state contains an out-of-range slot.');
+        indexes.add(index);
+        previous = index;
+        offset = decoded.offset;
+    }
+    return indexes;
+}
+
+export function encodeAdaptiveBitset(values, slotCount) {
+    const enabled = [];
+    const disabled = [];
+    const denseBytes = new Uint8Array(Math.ceil(slotCount / 8));
+    for (const [slot, value] of values) {
+        (value ? enabled : disabled).push(slot);
+        setBit(denseBytes, slot, value);
+    }
+    enabled.sort((left, right) => left - right);
+    disabled.sort((left, right) => left - right);
+    if (enabled.length === 0) return new Uint8Array();
+
+    const candidates = [
+        Uint8Array.from([0, ...trimTrailingZeroBytes(denseBytes)]),
+        Uint8Array.from([1, ...encodeSparseIndexes(enabled)]),
+        Uint8Array.from([2, ...encodeUnsignedVarint(slotCount), ...encodeSparseIndexes(disabled)])
+    ];
+    return candidates.reduce((shortest, candidate) =>
+        candidate.length < shortest.length ? candidate : shortest
+    );
+}
+
+export function decodeAdaptiveBitset(bytes, slotCount) {
+    if (bytes.length === 0) return () => false;
+    const mode = bytes[0];
+    if (mode === 0) {
+        const dense = bytes.slice(1);
+        if (dense.length > Math.ceil(slotCount / 8)) {
+            throw new Error('Dense tree state exceeds the manifest size.');
+        }
+        return slot => getBit(dense, slot);
+    }
+    if (mode === 1) {
+        const enabled = decodeSparseIndexes(bytes, 1, slotCount);
+        return slot => enabled.has(slot);
+    }
+    if (mode === 2) {
+        const coverage = decodeUnsignedVarint(bytes, 1);
+        if (coverage.value > slotCount) throw new Error('Tree-state coverage exceeds the manifest size.');
+        const disabled = decodeSparseIndexes(bytes, coverage.offset, coverage.value);
+        return slot => slot < coverage.value && !disabled.has(slot);
+    }
+    throw new Error('Unknown adaptive tree-state mode.');
+}
+
 function asManifest(manifest) {
     return manifest?.pathToSlot instanceof Map ? manifest : loadManifest(manifest);
 }
@@ -127,9 +223,8 @@ export function validateManifestEvolution(previous, next) {
 
 export function encodeTreeState(tree, manifest) {
     const normalized = asManifest(manifest);
-    const byteLength = Math.ceil(normalized.slots.length / 8);
-    const checked = new Uint8Array(byteLength);
-    const expanded = new Uint8Array(byteLength);
+    const checkedValues = new Map();
+    const expandedValues = new Map();
 
     for (const [path, node] of tree.nodesByPath) {
         const slot = normalized.pathToSlot.get(path);
@@ -137,12 +232,12 @@ export function encodeTreeState(tree, manifest) {
         const item = tree.itemForNode(node.id);
         const checkbox = item && tree.directCheckbox(item);
         const toggle = item?.querySelector(':scope > .checkbox-tree__row > .checkbox-tree__toggle');
-        setBit(checked, slot, Boolean(checkbox?.checked));
-        setBit(expanded, slot, toggle?.getAttribute('aria-expanded') === 'true');
+        checkedValues.set(slot, Boolean(checkbox?.checked));
+        expandedValues.set(slot, toggle?.getAttribute('aria-expanded') === 'true');
     }
 
-    const checkedValue = bytesToBase64Url(trimTrailingZeroBytes(checked));
-    const expandedValue = bytesToBase64Url(trimTrailingZeroBytes(expanded));
+    const checkedValue = bytesToBase64Url(encodeAdaptiveBitset(checkedValues, normalized.slots.length));
+    const expandedValue = bytesToBase64Url(encodeAdaptiveBitset(expandedValues, normalized.slots.length));
     return { version: SERIALIZATION_VERSION, checked: checkedValue || null, expanded: expandedValue || null };
 }
 
@@ -153,13 +248,20 @@ export function decodeTreeState(fragment, manifest) {
     if (params.get('v') !== String(SERIALIZATION_VERSION)) {
         return { applied: false, version: params.get('v'), warnings: ['Missing or unsupported serialization version.'] };
     }
-    const maximumLength = Math.ceil(normalized.slots.length / 8);
+    const maximumLength = Math.max(
+        Math.ceil(normalized.slots.length / 8) + 1,
+        normalized.slots.length * 2 + 10
+    );
     try {
+        const checked = base64UrlToBytes(params.get('c') ?? '', maximumLength);
+        const expanded = base64UrlToBytes(params.get('x') ?? '', maximumLength);
+        decodeAdaptiveBitset(checked, normalized.slots.length);
+        decodeAdaptiveBitset(expanded, normalized.slots.length);
         return {
             applied: true,
             version: SERIALIZATION_VERSION,
-            checked: base64UrlToBytes(params.get('c') ?? '', maximumLength),
-            expanded: base64UrlToBytes(params.get('x') ?? '', maximumLength),
+            checked,
+            expanded,
             warnings: []
         };
     } catch (error) {
@@ -170,13 +272,21 @@ export function decodeTreeState(fragment, manifest) {
 export function applyTreeState(tree, manifest, state) {
     if (!state?.applied) return false;
     const normalized = asManifest(manifest);
+    let isChecked;
+    let isExpanded;
+    try {
+        isChecked = decodeAdaptiveBitset(state.checked, normalized.slots.length);
+        isExpanded = decodeAdaptiveBitset(state.expanded, normalized.slots.length);
+    } catch {
+        return false;
+    }
     for (const [path, node] of tree.nodesByPath) {
         const slot = normalized.pathToSlot.get(path);
         if (slot === undefined) continue;
         const item = tree.itemForNode(node.id);
         const checkbox = item && tree.directCheckbox(item);
-        if (checkbox && !checkbox.disabled) checkbox.checked = getBit(state.checked, slot);
-        tree.setExpanded(item, getBit(state.expanded, slot));
+        if (checkbox && !checkbox.disabled) checkbox.checked = isChecked(slot);
+        tree.setExpanded(item, isExpanded(slot));
     }
     tree.updateAllParentCheckboxes();
     return true;

test/checkbox-tree.test.js

@@ -13,6 +13,8 @@ const {
     base64UrlToBytes,
     bytesToBase64Url,
     decodeTreeState,
+    decodeAdaptiveBitset,
+    encodeAdaptiveBitset,
     getBit,
     loadManifest,
     setBit,
@@ -136,6 +138,28 @@ test('Base64URL encoding is unpadded, URL-safe, and round trips', () => {
     assert.throws(() => base64UrlToBytes('not valid!'), /Invalid Base64URL/);
 });
 
+test('adaptive bitsets choose sparse enabled and disabled representations', () => {
+    const mostlyDisabled = new Map(Array.from(
+        { length: 628 },
+        (_, slot) => [slot, slot === 2 || slot === 400]
+    ));
+    const sparseEnabled = encodeAdaptiveBitset(mostlyDisabled, 628);
+    assert.equal(sparseEnabled[0], 1);
+    const enabled = decodeAdaptiveBitset(sparseEnabled, 628);
+    assert.equal(enabled(2), true);
+    assert.equal(enabled(3), false);
+    assert.equal(enabled(400), true);
+
+    const nearlyAllEnabled = new Map(Array.from({ length: 100 }, (_, slot) => [slot, slot !== 7]));
+    const sparseDisabled = encodeAdaptiveBitset(nearlyAllEnabled, 100);
+    assert.equal(sparseDisabled[0], 2);
+    const enabledExcept = decodeAdaptiveBitset(sparseDisabled, 105);
+    assert.equal(enabledExcept(6), true);
+    assert.equal(enabledExcept(7), false);
+    assert.equal(enabledExcept(99), true);
+    assert.equal(enabledExcept(100), false);
+});
+
 test('loads text and JSON manifests while retaining tombstone slots', () => {
     const text = loadManifest('# tree-state-manifest\nroot\n!deleted\nroot>child\n');
     assert.equal(text.slots.length, 3);
@@ -170,7 +194,7 @@ test('serializes and restores checked and expanded state through a fragment', ()
     checkbox(first.container, 'first').click();
     first.container.querySelector('.checkbox-tree__toggle').click();
     const fragment = first.tree.serializeStateToFragment(manifest);
-    assert.equal(fragment, '#v=1&c=Ag&x=AQ');
+    assert.equal(fragment, '#v=1&c=AAI&x=AAE');
 
     const second = createTree({ manifest });
     const result = second.tree.restoreStateFromLocation(manifest, { hash: fragment });
@@ -201,7 +225,7 @@ test('malformed, oversized, and unsupported URL state fails safely', () => {
     const manifest = loadManifest({ schema: 1, nodes: ['parent', 'parent>first', 'parent>second'] });
     assert.equal(decodeTreeState('#v=2&c=AQ', manifest).applied, false);
     assert.equal(decodeTreeState('#v=1&c=!', manifest).applied, false);
-    assert.equal(decodeTreeState('#v=1&c=AQA', manifest).applied, false);
+    assert.equal(decodeTreeState('#v=1&c=Aw', manifest).applied, false);
 
     const { tree } = createTree({ initiallySelected: true });
     const result = tree.restoreStateFromLocation(manifest, { hash: '#v=1&c=!' });