diff --git a/crates/av1/tools/extract_tables.py b/crates/av1/tools/extract_tables.py index e64a931f1..a037a3765 100644 --- a/crates/av1/tools/extract_tables.py +++ b/crates/av1/tools/extract_tables.py @@ -49,7 +49,7 @@ def ev(expr): # --- constants: symbol table (section 3) --- pending = [] for line in texts["03.symbols.md"].split("\n"): - m = re.match(r"^\|\s*`([A-Z][A-Z0-9_]*)`\s*\|\s*([^|]+?)\s*\|", line) + m = re.match(r"^\|\s*`([A-Z][A-Z0-9_]*)`\s*\|\s*([^\s|](?:[^|]*[^|\s])?)\s*\|", line) if m: pending.append((m.group(1), m.group(2))) # --- constants: value/name tables (section 6 semantics, section 8/9) --- @@ -91,7 +91,7 @@ for name, val in pending: def cells(text): """Split one initialiser cell. A missing comma between two operands on separate lines (Split_Tx_Size in the spec text: `TX_32X32` / `TX_4X8`) yields two cells.""" - parts = re.split(r"(?<=[A-Za-z0-9_])\s*\n\s*(?=[A-Za-z0-9_])", text.strip()) + parts = re.split(r"(?<=[A-Za-z0-9_])[^\S\n]*\n[^\S\n]*(?=[A-Za-z0-9_])", text.strip()) if len(parts) > 1: sys.stderr.write("note: missing comma in spec table between %s\n" % parts) return parts @@ -132,10 +132,25 @@ def shape(x): return [] +# `~~~~` opens a block on its own line (with an optional `c` marker) and the next run of +# tildes closes it. Walk the pairs with two linear searches instead of the lazy `(.*?)` +# with re.S, which rescans the whole remaining text at every position (super-linear). +FENCE_OPEN = re.compile(r"~~~~~?\s*c?\n") +FENCE_CLOSE = re.compile(r"~~~~~?") + tables = {} for f in files: t = texts[f] - for block in re.findall(r"~~~~~?\s*c?\n(.*?)~~~~~?", t, re.S): + pos = 0 + while True: + m = FENCE_OPEN.search(t, pos) + if not m: + break + c = FENCE_CLOSE.search(t, m.end()) + if not c: + break + block = t[m.end():c.start()] + pos = c.end() b = re.sub(r"//[^\n]*", "", block) b = re.sub(r"/\*.*?\*/", "", b, flags=re.S) for m in re.finditer(r"(?m)^\s*([A-Z][A-Za-z0-9_]*)\s*((?:\[[^\]=]*\]\s*)+)=\s*", b): diff --git a/crates/hevc/tools/extract_tables.py b/crates/hevc/tools/extract_tables.py index 6ee5759df..8b7d7d35e 100644 --- a/crates/hevc/tools/extract_tables.py +++ b/crates/hevc/tools/extract_tables.py @@ -44,7 +44,7 @@ def body_after(title_regex, stop_regex=r"^\s*(Table \d+-\d+ - |\d+\s+Rec\. ITU|R def init_values(table): vals = [] for l in body_after(r"Table %s - Values of initValue" % re.escape(table)): - m = re.match(r"^\s*initValue\s+(.*)$", l) + m = re.match(r"^\s*initValue\s+(\S.*)$", l) if m: vals += [int(v) for v in m.group(1).split()] return vals @@ -118,7 +118,7 @@ w("") # Table 7-6: default 8x8 scaling lists (intra matrixId 0..2, inter 3..5) intra, inter = [], [] for l in body_after(r"Table 7-6 - Specification of default values of ScalingList"): - m = re.match(r"^\s*ScalingList\[ 1\.\.3 \]\[ (0\.\.2|3\.\.5) \]\[ i \]\s+(.*)$", l) + m = re.match(r"^\s*ScalingList\[ 1\.\.3 \]\[ (0\.\.2|3\.\.5) \]\[ i \]\s+(\S.*)$", l) if m: (intra if m.group(1) == "0..2" else inter).extend(int(v) for v in m.group(2).split()) assert len(intra) == 64 and len(inter) == 64