# unescape_html in Nim with Wasm SIMD: decodes every HTML character reference in UTF-8 text
# the way Python's html.unescape does, and copies all other bytes unchanged.
#
# The design is the one from my C and WebAssembly text entries:
# - 64 zero bytes are written after the input, so no load or loop needs a bounds check.
# - Two stages per 4 KiB window. Stage 1 turns each 64-byte block into a bitmask of its `&`
# and appends their positions to a list. Stage 2 walks the list: copy the text since the
# last reference (32 bytes unconditionally, more in a loop), then decode.
# - Inputs without `&` are returned where they are; nothing is copied.
# - A name of up to 7 bytes ending in `;` is looked up without measuring it: the first `;`
# in the next 8 bytes is found with a SWAR zero-byte test, the bytes up to it are the key,
# and one multiply indexes an open-addressed table of 16-byte entries (key, value).
# - Longer names go through a second table and one masked 32-byte vector compare.
# - Without `;` only the 106 legacy names match. A filter on the first two bytes gives the
# longest legacy name that could start there, usually none.
# - Numeric references of up to 7 digits, decimal or hexadecimal, share one branch-free
# SWAR path.
# - Output and input share one buffer: output from its start, input 256 KiB in. A reference
# grows by at most 1.2x (`≫⃒` is 5 bytes in, 6 out), so the write position stays at
# least 51 KiB behind the read position and decoding in place is safe.
# - The entity list is stored packed (5.9 KB for 2,125 names) and unpacked on the first
# call: see `sym` and `init`. pack_mix.py, at the end of this file, is the matching packer.
#
# Build: see the end of this file.
{.push checks: off, stackTrace: off, lineTrace: off.}
const
InputCap = 1'u32 shl 20
# The largest growth is `≫⃒` (5 bytes) becoming 6 bytes.
OutputCap = InputCap div 5 * 6 + 8
InOff = 1'u32 shl 18
Pad = 128
FastBits = 12
FastTail = 32
LegBits = 8
LegTail = 16
LongBits = 11
Empty = not 0'u64
# Bytes of input per stage-1 window; a multiple of 64.
Window = 4096'u32
Ones = 0x0101010101010101'u64
High = 0x8080808080808080'u64
K = 1'u32 shl 20
const
HistSize = 16130
EntityCount = 2125
LongCount = 506
ModelBits = 17
ModelLimit = 3
MixShift = 11
MixW0 = 16384
FastMul = 0xa59d054b12f5b905'u64
LegMul = 0xf880d97d1854e551'u64
LongMul = 0xa0c2995f40498cb3'u64
# e^(2047 / 256) and e^(-1 / 256) as the bits of a float64.
SquashE0 = 0x40a732ac33de5528'u64
SquashStep = 0x3fefe00ffaabffbc'u64
# Step toward the bit just seen, as a 16-bit fraction: 2 / (2n + 3) after n uses.
rate: array[4, uint16] = [43690'u16, 26214'u16, 18724'u16, 14563'u16]
packed: array[5870, uint8] = [
222'u8, 225, 76, 127, 180, 26, 183, 40, 117, 111, 212, 1, 220, 243, 80, 19, 137, 147, 125, 129, 7, 7, 137, 16,
130, 158, 138, 125, 5, 234, 196, 106, 89, 69, 197, 243, 136, 86, 218, 152, 109, 108, 69, 33, 55, 71, 159, 88,
85, 51, 112, 51, 107, 141, 51, 22, 154, 180, 33, 140, 63, 154, 131, 152, 13, 94, 30, 228, 23, 251, 135, 236,
129, 186, 220, 6, 97, 11, 99, 33, 221, 217, 224, 75, 142, 69, 12, 159, 171, 25, 206, 254, 134, 114, 176, 175,
132, 228, 146, 96, 98, 30, 154, 222, 150, 71, 243, 66, 148, 174, 133, 30, 142, 131, 177, 159, 133, 130, 160, 123,
193, 128, 149, 204, 238, 192, 189, 17, 170, 157, 176, 106, 63, 181, 103, 100, 115, 245, 163, 25, 48, 120, 48, 82,
90, 172, 100, 47, 144, 202, 122, 72, 163, 117, 75, 124, 120, 117, 22, 7, 14, 170, 223, 96, 10, 146, 77, 173,
18, 191, 89, 4, 244, 115, 7, 79, 210, 9, 33, 56, 59, 215, 241, 153, 59, 36, 189, 73, 82, 224, 139, 90,
93, 19, 211, 248, 230, 217, 167, 18, 133, 137, 245, 198, 55, 115, 75, 158, 211, 214, 26, 9, 87, 209, 211, 4,
177, 236, 186, 241, 16, 10, 14, 67, 206, 150, 88, 249, 60, 167, 107, 104, 183, 1, 137, 162, 179, 3, 59, 221,
152, 48, 54, 68, 233, 158, 100, 183, 147, 198, 110, 208, 213, 22, 150, 10, 176, 18, 116, 190, 192, 78, 15, 48,
55, 236, 30, 217, 23, 70, 169, 50, 20, 82, 52, 164, 137, 115, 232, 112, 101, 176, 1, 67, 149, 92, 57, 181,
66, 17, 233, 11, 129, 139, 113, 178, 135, 47, 253, 143, 132, 85, 68, 186, 72, 3, 103, 150, 211, 8, 179, 168,
70, 203, 65, 169, 107, 64, 171, 34, 180, 33, 131, 16, 18, 136, 202, 27, 222, 53, 101, 51, 101, 248, 245, 158,
120, 74, 8, 80, 20, 145, 180, 26, 30, 60, 134, 122, 111, 45, 198, 61, 18, 171, 161, 154, 110, 66, 232, 199,
244, 123, 40, 192, 27, 47, 72, 141, 122, 210, 195, 57, 122, 193, 235, 51, 242, 98, 133, 36, 66, 231, 30, 152,
141, 128, 122, 49, 254, 46, 254, 29, 158, 140, 157, 104, 190, 7, 159, 239, 249, 30, 135, 181, 86, 112, 132, 137,
12, 161, 74, 143, 90, 113, 214, 94, 84, 174, 167, 164, 51, 12, 134, 76, 84, 208, 162, 167, 228, 152, 54, 217,
149, 137, 111, 213, 237, 235, 195, 114, 209, 114, 18, 152, 206, 97, 67, 74, 142, 199, 62, 128, 137, 60, 3, 38,
70, 140, 231, 253, 149, 133, 239, 165, 223, 61, 165, 184, 244, 24, 157, 223, 170, 85, 3, 191, 64, 75, 86, 23,
75, 120, 85, 159, 183, 50, 244, 27, 189, 1, 110, 61, 214, 183, 159, 85, 181, 194, 35, 12, 29, 30, 215, 239,
145, 231, 233, 219, 229, 61, 192, 194, 212, 164, 45, 127, 127, 201, 117, 178, 249, 165, 18, 46, 117, 145, 19, 48,
247, 250, 227, 61, 25, 22, 103, 32, 68, 206, 47, 236, 169, 202, 175, 228, 179, 213, 254, 111, 10, 62, 177, 47,
77, 103, 156, 229, 98, 75, 47, 226, 223, 136, 169, 10, 9, 71, 114, 20, 103, 190, 241, 119, 64, 253, 233, 243,
218, 81, 217, 215, 12, 105, 31, 29, 137, 211, 209, 148, 207, 223, 153, 70, 136, 242, 220, 103, 247, 161, 203, 217,
212, 126, 218, 98, 232, 107, 247, 116, 230, 194, 84, 137, 6, 65, 201, 239, 2, 42, 134, 114, 1, 141, 237, 208,
193, 126, 156, 218, 72, 226, 48, 249, 64, 246, 19, 44, 83, 105, 161, 79, 131, 144, 132, 90, 217, 234, 188, 193,
118, 55, 241, 15, 221, 5, 113, 10, 89, 208, 167, 156, 137, 242, 120, 178, 86, 177, 39, 165, 240, 64, 42, 217,
43, 5, 179, 28, 220, 244, 197, 135, 196, 131, 109, 185, 88, 158, 47, 125, 233, 98, 86, 13, 110, 112, 47, 47,
10, 71, 46, 7, 201, 184, 88, 61, 187, 201, 187, 209, 159, 93, 175, 132, 52, 115, 232, 1, 80, 109, 122, 159,
40, 42, 41, 241, 48, 14, 9, 168, 172, 16, 43, 235, 82, 34, 246, 61, 92, 150, 95, 200, 223, 24, 185, 156,
187, 170, 139, 27, 248, 230, 66, 157, 150, 44, 248, 160, 131, 147, 137, 156, 107, 181, 47, 13, 27, 69, 148, 78,
85, 182, 253, 66, 6, 23, 251, 35, 166, 230, 199, 154, 112, 3, 57, 160, 119, 137, 28, 136, 73, 191, 146, 66,
144, 168, 90, 27, 98, 153, 184, 94, 121, 124, 229, 18, 219, 227, 138, 175, 83, 236, 63, 217, 199, 223, 181, 138,
223, 213, 207, 98, 28, 31, 239, 237, 100, 75, 50, 131, 123, 122, 107, 70, 127, 125, 120, 119, 69, 176, 143, 200,
46, 88, 53, 216, 233, 182, 181, 157, 204, 132, 19, 222, 6, 199, 113, 223, 208, 84, 11, 129, 53, 39, 85, 186,
203, 216, 236, 241, 249, 53, 18, 204, 218, 166, 115, 248, 145, 198, 46, 206, 254, 174, 217, 232, 138, 92, 123, 120,
71, 48, 58, 72, 157, 6, 170, 203, 94, 173, 53, 195, 84, 15, 225, 240, 64, 220, 236, 204, 105, 54, 97, 109,
17, 73, 188, 216, 230, 42, 13, 126, 212, 248, 225, 216, 2, 3, 96, 141, 173, 92, 235, 176, 145, 62, 114, 173,
212, 21, 11, 227, 218, 13, 59, 33, 123, 220, 142, 130, 58, 197, 213, 118, 232, 190, 78, 35, 109, 249, 15, 248,
16, 236, 150, 185, 238, 155, 31, 222, 70, 254, 82, 244, 88, 149, 57, 16, 73, 8, 204, 72, 57, 236, 170, 54,
26, 184, 248, 155, 85, 42, 85, 204, 117, 124, 22, 129, 51, 71, 158, 255, 206, 26, 97, 178, 128, 246, 40, 217,
85, 194, 0, 166, 4, 119, 243, 33, 240, 155, 110, 206, 40, 164, 247, 74, 145, 55, 252, 114, 131, 33, 187, 127,
204, 42, 245, 20, 11, 24, 107, 200, 9, 21, 168, 165, 226, 210, 67, 186, 156, 112, 90, 140, 10, 228, 176, 168,
26, 170, 42, 201, 82, 171, 112, 111, 245, 233, 118, 235, 168, 27, 36, 52, 50, 0, 59, 12, 207, 40, 210, 111,
146, 243, 162, 129, 76, 136, 247, 122, 35, 184, 253, 173, 227, 29, 228, 185, 235, 128, 52, 243, 242, 236, 78, 59,
239, 195, 116, 252, 229, 116, 151, 85, 2, 1, 23, 247, 108, 6, 198, 139, 164, 24, 110, 0, 63, 51, 238, 37,
33, 24, 161, 238, 202, 87, 226, 31, 14, 9, 221, 31, 144, 44, 55, 234, 140, 17, 13, 153, 233, 45, 28, 50,
86, 185, 183, 181, 122, 108, 174, 12, 224, 81, 12, 207, 69, 124, 197, 114, 2, 81, 167, 104, 133, 241, 147, 234,
129, 130, 169, 161, 254, 89, 28, 0, 214, 48, 34, 97, 181, 246, 67, 135, 250, 120, 9, 43, 218, 52, 126, 45,
251, 58, 155, 240, 209, 154, 37, 216, 245, 152, 203, 1, 192, 255, 183, 144, 16, 198, 67, 78, 104, 252, 149, 214,
59, 222, 219, 84, 226, 85, 179, 241, 68, 67, 52, 125, 148, 156, 228, 88, 92, 188, 37, 42, 117, 160, 242, 156,
234, 153, 217, 242, 120, 244, 47, 115, 114, 150, 90, 177, 199, 30, 178, 56, 225, 251, 184, 103, 116, 115, 122, 163,
238, 7, 80, 203, 155, 205, 201, 151, 216, 154, 83, 237, 98, 89, 146, 41, 66, 250, 180, 123, 111, 87, 185, 30,
135, 179, 214, 53, 53, 49, 40, 182, 48, 229, 87, 133, 236, 42, 149, 15, 13, 45, 18, 128, 116, 61, 106, 142,
217, 86, 245, 243, 251, 219, 224, 75, 102, 16, 46, 80, 27, 157, 55, 27, 204, 224, 172, 4, 164, 194, 147, 218,
96, 158, 211, 122, 125, 169, 55, 43, 82, 252, 58, 154, 32, 114, 205, 76, 251, 0, 107, 216, 87, 65, 40, 207,
174, 212, 200, 62, 108, 187, 144, 162, 144, 33, 235, 174, 126, 89, 151, 233, 3, 236, 121, 107, 246, 222, 10, 108,
83, 250, 144, 37, 98, 3, 49, 236, 83, 101, 71, 120, 49, 206, 186, 242, 126, 199, 94, 28, 50, 151, 146, 231,
249, 112, 152, 25, 118, 184, 19, 27, 212, 17, 181, 108, 83, 125, 105, 173, 149, 114, 46, 131, 59, 63, 243, 48,
51, 91, 77, 177, 46, 63, 129, 239, 14, 153, 255, 94, 196, 45, 254, 105, 95, 71, 145, 23, 101, 28, 28, 43,
253, 71, 28, 203, 37, 96, 103, 233, 85, 127, 239, 37, 120, 244, 55, 121, 29, 192, 92, 51, 161, 203, 113, 148,
154, 133, 241, 132, 70, 121, 86, 81, 178, 32, 156, 142, 103, 87, 229, 150, 222, 186, 162, 252, 233, 243, 193, 18,
189, 53, 168, 91, 155, 234, 208, 215, 116, 89, 125, 232, 71, 187, 126, 244, 67, 93, 147, 95, 190, 106, 130, 124,
16, 65, 75, 43, 219, 206, 86, 36, 228, 214, 91, 4, 238, 223, 120, 80, 125, 100, 33, 92, 155, 217, 150, 31,
164, 5, 166, 234, 135, 57, 22, 227, 75, 56, 244, 89, 52, 39, 158, 67, 21, 71, 76, 235, 221, 112, 17, 242,
44, 235, 250, 78, 84, 191, 29, 87, 28, 16, 211, 178, 109, 159, 174, 68, 53, 253, 188, 51, 226, 93, 181, 80,
151, 239, 11, 63, 96, 243, 228, 67, 10, 92, 162, 52, 120, 159, 20, 105, 194, 155, 69, 68, 71, 119, 128, 152,
224, 242, 192, 20, 50, 139, 252, 90, 215, 30, 39, 193, 61, 230, 19, 99, 0, 193, 85, 44, 45, 242, 225, 148,
214, 8, 7, 98, 129, 55, 222, 177, 227, 3, 238, 125, 152, 238, 152, 252, 92, 106, 185, 142, 193, 153, 59, 123,
106, 117, 189, 78, 210, 53, 53, 89, 226, 255, 132, 242, 74, 60, 146, 223, 49, 237, 59, 243, 52, 164, 225, 146,
108, 91, 29, 57, 154, 199, 111, 17, 53, 20, 193, 60, 69, 151, 214, 232, 116, 160, 113, 120, 99, 137, 230, 175,
232, 162, 29, 8, 26, 164, 19, 164, 75, 14, 47, 78, 175, 232, 251, 58, 7, 47, 196, 183, 255, 149, 94, 234,
12, 177, 64, 200, 210, 61, 73, 17, 147, 211, 3, 49, 98, 252, 44, 17, 207, 23, 10, 133, 220, 164, 111, 204,
227, 141, 207, 182, 152, 20, 14, 242, 96, 12, 79, 106, 236, 191, 235, 66, 139, 214, 100, 146, 223, 171, 56, 69,
8, 98, 31, 111, 3, 237, 206, 41, 159, 182, 153, 32, 203, 231, 70, 118, 203, 193, 19, 176, 78, 187, 167, 86,
248, 118, 7, 221, 59, 106, 33, 78, 150, 5, 153, 20, 82, 165, 77, 114, 122, 66, 105, 147, 206, 176, 9, 243,
178, 52, 91, 165, 190, 24, 56, 43, 133, 95, 118, 45, 152, 140, 67, 250, 196, 173, 222, 60, 223, 184, 102, 56,
44, 197, 8, 168, 18, 114, 248, 160, 23, 217, 201, 103, 99, 92, 137, 128, 215, 19, 77, 242, 35, 250, 218, 122,
41, 124, 20, 5, 108, 127, 47, 240, 93, 255, 239, 121, 103, 144, 59, 114, 183, 173, 215, 163, 210, 112, 207, 235,
170, 22, 9, 100, 75, 4, 9, 150, 186, 137, 0, 215, 132, 129, 215, 79, 99, 173, 165, 223, 232, 23, 69, 125,
6, 77, 8, 201, 221, 148, 8, 156, 156, 232, 19, 204, 19, 119, 89, 214, 75, 183, 131, 6, 140, 16, 79, 24,
92, 175, 71, 162, 141, 143, 205, 182, 49, 136, 242, 241, 3, 205, 204, 69, 186, 181, 53, 126, 248, 239, 31, 97,
122, 161, 3, 116, 152, 209, 247, 61, 181, 45, 205, 187, 11, 175, 5, 32, 105, 45, 229, 97, 157, 219, 147, 22,
57, 64, 168, 224, 124, 10, 2, 27, 10, 198, 72, 133, 239, 113, 21, 98, 67, 37, 209, 153, 148, 229, 49, 158,
143, 48, 56, 231, 181, 119, 145, 224, 78, 1, 233, 90, 81, 85, 243, 8, 150, 158, 67, 169, 120, 90, 171, 104,
177, 169, 102, 92, 76, 165, 224, 241, 80, 98, 170, 164, 254, 19, 36, 80, 144, 155, 222, 254, 33, 74, 43, 238,
28, 177, 205, 15, 238, 160, 228, 228, 156, 179, 135, 251, 142, 167, 158, 9, 8, 191, 174, 223, 169, 234, 13, 205,
107, 133, 250, 78, 178, 66, 32, 1, 146, 7, 35, 102, 60, 31, 36, 93, 218, 194, 3, 3, 54, 3, 35, 143,
108, 117, 74, 125, 132, 149, 228, 88, 214, 172, 110, 165, 115, 237, 31, 253, 131, 128, 232, 114, 196, 151, 107, 108,
52, 189, 180, 86, 37, 221, 111, 31, 221, 98, 13, 185, 1, 226, 144, 68, 192, 134, 51, 241, 121, 208, 121, 63,
196, 139, 186, 98, 234, 110, 27, 47, 248, 58, 184, 99, 185, 22, 167, 174, 170, 107, 234, 59, 76, 3, 110, 205,
69, 201, 40, 212, 169, 88, 93, 104, 182, 59, 68, 82, 121, 182, 239, 196, 123, 252, 160, 32, 147, 101, 143, 225,
16, 72, 65, 216, 190, 235, 114, 240, 64, 230, 155, 228, 160, 165, 244, 38, 43, 237, 206, 149, 31, 170, 175, 179,
44, 162, 154, 201, 224, 250, 87, 173, 20, 108, 66, 10, 127, 154, 81, 189, 202, 126, 0, 61, 232, 189, 52, 105,
245, 153, 117, 165, 45, 242, 79, 141, 39, 97, 11, 18, 121, 16, 137, 59, 247, 56, 158, 143, 22, 6, 0, 59,
226, 48, 37, 243, 190, 180, 254, 79, 196, 225, 199, 56, 5, 94, 177, 222, 103, 11, 192, 72, 63, 137, 90, 250,
232, 179, 75, 47, 221, 108, 137, 197, 239, 31, 127, 154, 142, 239, 208, 249, 240, 51, 38, 246, 66, 213, 108, 60,
104, 220, 169, 76, 134, 214, 146, 233, 145, 234, 200, 14, 87, 87, 65, 193, 46, 10, 213, 45, 37, 172, 94, 119,
44, 7, 213, 151, 103, 171, 51, 7, 162, 78, 36, 178, 174, 115, 156, 192, 159, 244, 133, 161, 137, 136, 252, 139,
61, 159, 175, 67, 68, 43, 50, 62, 161, 32, 28, 74, 242, 139, 85, 33, 194, 1, 173, 45, 220, 67, 34, 176,
11, 108, 129, 133, 26, 105, 200, 68, 38, 129, 244, 168, 67, 223, 58, 103, 93, 242, 75, 181, 189, 116, 207, 159,
22, 48, 30, 73, 148, 85, 254, 2, 45, 144, 128, 170, 80, 94, 175, 210, 177, 22, 41, 168, 170, 26, 237, 5,
0, 148, 252, 132, 79, 204, 230, 179, 31, 223, 232, 37, 185, 23, 247, 140, 85, 202, 183, 92, 227, 81, 91, 107,
157, 1, 56, 158, 231, 139, 245, 113, 164, 203, 206, 230, 127, 179, 83, 223, 101, 226, 227, 222, 190, 37, 172, 75,
222, 21, 126, 107, 170, 87, 207, 139, 140, 179, 185, 83, 110, 230, 188, 248, 208, 82, 25, 211, 242, 160, 59, 157,
226, 0, 0, 18, 214, 140, 110, 191, 14, 121, 79, 94, 10, 196, 253, 10, 18, 232, 193, 176, 239, 183, 101, 172,
125, 240, 233, 155, 39, 52, 39, 228, 186, 146, 226, 10, 132, 90, 24, 0, 147, 141, 99, 172, 8, 175, 157, 243,
171, 223, 100, 175, 214, 149, 124, 121, 218, 233, 253, 251, 153, 189, 59, 227, 156, 100, 158, 126, 165, 10, 150, 192,
27, 153, 176, 152, 41, 32, 229, 92, 218, 171, 107, 38, 58, 193, 132, 77, 69, 147, 153, 105, 41, 123, 226, 87,
39, 148, 206, 185, 73, 249, 142, 149, 216, 82, 22, 126, 89, 221, 141, 188, 46, 73, 25, 122, 253, 80, 199, 12,
180, 179, 200, 141, 190, 80, 156, 36, 61, 168, 8, 129, 124, 67, 62, 123, 52, 87, 151, 234, 4, 59, 253, 94,
144, 62, 81, 42, 48, 214, 181, 15, 239, 161, 92, 209, 212, 162, 235, 43, 48, 231, 53, 253, 217, 110, 97, 188,
208, 250, 248, 59, 211, 28, 213, 73, 70, 199, 148, 67, 95, 29, 15, 30, 209, 70, 126, 181, 85, 245, 180, 26,
246, 230, 53, 39, 53, 49, 92, 14, 87, 39, 73, 140, 127, 155, 170, 1, 140, 204, 126, 133, 225, 137, 106, 187,
245, 70, 162, 52, 250, 88, 207, 14, 239, 83, 7, 66, 134, 229, 176, 139, 221, 78, 49, 30, 209, 120, 127, 67,
1, 134, 29, 146, 175, 160, 177, 141, 110, 189, 117, 191, 148, 109, 255, 70, 202, 252, 195, 111, 91, 67, 167, 90,
96, 8, 148, 215, 27, 63, 59, 4, 164, 212, 111, 101, 11, 249, 171, 96, 140, 196, 164, 32, 222, 0, 182, 148,
233, 195, 237, 38, 188, 107, 19, 151, 229, 108, 238, 144, 194, 119, 7, 221, 213, 36, 23, 107, 157, 177, 13, 242,
110, 59, 108, 115, 242, 151, 228, 227, 56, 78, 3, 218, 233, 227, 171, 127, 156, 78, 66, 175, 185, 124, 180, 240,
113, 193, 212, 97, 22, 245, 220, 33, 28, 161, 193, 36, 99, 27, 32, 249, 57, 169, 122, 102, 213, 209, 198, 151,
207, 171, 129, 38, 35, 105, 79, 230, 246, 166, 167, 5, 2, 139, 197, 11, 151, 121, 228, 104, 244, 247, 113, 57,
86, 87, 252, 141, 250, 138, 194, 101, 91, 242, 48, 238, 117, 138, 52, 188, 105, 141, 167, 206, 113, 8, 212, 176,
94, 131, 200, 101, 43, 92, 156, 6, 26, 170, 212, 133, 146, 141, 150, 34, 51, 253, 147, 98, 143, 75, 40, 199,
210, 34, 99, 155, 224, 90, 71, 129, 70, 211, 196, 21, 2, 66, 74, 179, 240, 11, 60, 178, 158, 197, 26, 6,
211, 98, 155, 47, 120, 216, 2, 206, 243, 165, 37, 227, 159, 171, 171, 213, 88, 244, 80, 253, 138, 208, 180, 237,
154, 31, 225, 88, 49, 160, 111, 90, 219, 141, 193, 220, 56, 9, 47, 46, 85, 254, 170, 193, 122, 17, 35, 230,
224, 193, 219, 60, 120, 204, 80, 200, 206, 205, 167, 45, 80, 222, 111, 95, 243, 23, 65, 194, 54, 41, 129, 25,
74, 49, 252, 233, 197, 77, 195, 78, 142, 219, 172, 73, 46, 238, 90, 215, 50, 86, 70, 159, 35, 188, 207, 171,
4, 89, 255, 249, 13, 217, 104, 16, 75, 96, 36, 230, 158, 42, 113, 165, 95, 144, 35, 213, 188, 88, 11, 19,
41, 99, 151, 62, 57, 118, 103, 130, 46, 176, 210, 145, 141, 142, 110, 174, 113, 159, 17, 162, 28, 48, 23, 250,
135, 218, 17, 254, 126, 39, 0, 69, 177, 54, 122, 57, 243, 45, 149, 149, 227, 3, 213, 51, 23, 25, 189, 105,
108, 238, 43, 103, 52, 71, 58, 72, 220, 229, 160, 90, 86, 184, 54, 207, 211, 233, 93, 79, 191, 71, 193, 129,
251, 138, 103, 144, 172, 6, 190, 118, 231, 129, 213, 103, 187, 224, 205, 97, 101, 19, 164, 66, 128, 53, 31, 188,
11, 194, 161, 251, 95, 218, 225, 85, 52, 82, 151, 81, 5, 191, 89, 243, 46, 98, 21, 0, 22, 43, 76, 1,
124, 230, 163, 71, 115, 19, 175, 137, 128, 164, 225, 111, 89, 253, 124, 172, 60, 54, 35, 72, 97, 194, 4, 8,
97, 84, 137, 141, 221, 140, 91, 191, 49, 27, 44, 66, 254, 11, 122, 161, 16, 134, 174, 200, 131, 95, 36, 146,
56, 189, 136, 144, 32, 168, 7, 97, 209, 222, 67, 255, 71, 174, 132, 254, 29, 239, 58, 175, 41, 161, 174, 48,
67, 194, 24, 193, 172, 183, 14, 127, 145, 32, 226, 75, 251, 146, 180, 205, 75, 183, 70, 107, 219, 43, 130, 212,
101, 238, 255, 140, 14, 102, 46, 183, 129, 225, 227, 156, 56, 7, 92, 67, 13, 4, 22, 103, 41, 28, 69, 170,
203, 36, 213, 22, 155, 23, 52, 84, 112, 22, 42, 1, 81, 32, 95, 98, 8, 59, 239, 61, 103, 183, 226, 5,
245, 19, 175, 246, 1, 20, 186, 177, 202, 97, 185, 217, 61, 254, 190, 220, 211, 144, 151, 39, 247, 62, 109, 151,
102, 40, 142, 99, 120, 233, 62, 244, 210, 221, 63, 174, 22, 34, 105, 72, 32, 103, 226, 194, 80, 31, 190, 201,
133, 40, 0, 13, 88, 99, 90, 203, 251, 10, 194, 111, 255, 100, 75, 211, 198, 70, 43, 226, 175, 2, 101, 74,
198, 104, 136, 67, 139, 123, 238, 109, 196, 103, 44, 243, 0, 120, 131, 170, 233, 68, 18, 124, 86, 175, 242, 204,
89, 139, 75, 146, 26, 167, 125, 176, 222, 36, 123, 81, 188, 96, 168, 151, 55, 231, 103, 108, 88, 231, 50, 5,
103, 1, 130, 97, 125, 228, 4, 236, 7, 162, 44, 190, 199, 163, 212, 60, 24, 244, 18, 155, 139, 114, 242, 243,
164, 238, 94, 62, 151, 63, 124, 87, 207, 169, 34, 91, 22, 73, 184, 18, 211, 31, 229, 74, 6, 66, 1, 159,
128, 31, 114, 248, 157, 0, 86, 5, 255, 80, 132, 76, 146, 115, 198, 127, 50, 154, 83, 139, 199, 130, 87, 153,
232, 133, 94, 162, 166, 235, 114, 108, 236, 13, 45, 17, 47, 71, 2, 241, 45, 183, 12, 29, 83, 88, 176, 171,
199, 26, 29, 250, 246, 159, 35, 216, 249, 169, 140, 195, 187, 57, 116, 244, 245, 166, 136, 55, 113, 2, 101, 112,
33, 187, 141, 200, 207, 116, 143, 71, 9, 78, 124, 103, 150, 236, 251, 119, 18, 247, 10, 58, 246, 244, 71, 151,
156, 151, 245, 56, 115, 69, 187, 138, 47, 97, 200, 238, 5, 111, 114, 119, 87, 142, 5, 220, 2, 232, 2, 196,
204, 91, 18, 72, 58, 88, 248, 52, 11, 141, 227, 77, 185, 131, 32, 115, 52, 8, 48, 10, 183, 218, 225, 202,
193, 202, 122, 152, 250, 16, 166, 12, 192, 41, 113, 3, 198, 44, 89, 247, 107, 86, 193, 162, 9, 152, 12, 197,
236, 37, 136, 222, 144, 234, 33, 55, 204, 218, 28, 109, 43, 34, 8, 194, 237, 52, 245, 11, 141, 173, 27, 56,
216, 63, 103, 81, 40, 226, 222, 180, 20, 147, 118, 218, 8, 223, 91, 109, 3, 169, 239, 244, 11, 58, 17, 201,
192, 56, 86, 84, 31, 90, 29, 200, 153, 240, 8, 65, 238, 14, 52, 246, 254, 85, 167, 168, 176, 206, 44, 153,
200, 122, 129, 230, 113, 103, 27, 177, 162, 146, 169, 91, 36, 188, 249, 157, 175, 9, 38, 234, 160, 161, 158, 39,
93, 188, 166, 254, 251, 216, 149, 162, 134, 219, 175, 111, 208, 164, 43, 87, 37, 109, 84, 6, 41, 170, 180, 219,
211, 64, 13, 119, 45, 232, 212, 45, 116, 114, 158, 210, 248, 108, 13, 243, 232, 106, 250, 34, 209, 169, 187, 25,
241, 69, 139, 15, 1, 53, 134, 77, 211, 239, 206, 181, 247, 143, 69, 182, 73, 248, 55, 129, 96, 55, 81, 171,
75, 115, 61, 219, 205, 169, 138, 46, 90, 249, 184, 241, 199, 235, 87, 53, 83, 243, 9, 108, 244, 157, 158, 36,
244, 67, 244, 89, 59, 169, 184, 209, 37, 184, 190, 141, 125, 4, 74, 9, 142, 206, 181, 252, 155, 43, 254, 15,
19, 124, 112, 104, 12, 90, 87, 44, 34, 103, 75, 64, 1, 84, 17, 214, 78, 167, 101, 149, 67, 97, 245, 186,
123, 125, 18, 252, 141, 246, 148, 233, 236, 187, 232, 170, 139, 224, 113, 162, 0, 43, 160, 9, 71, 69, 142, 123,
40, 250, 25, 187, 28, 29, 165, 241, 48, 42, 209, 185, 97, 35, 171, 45, 124, 96, 109, 251, 20, 181, 211, 176,
125, 115, 88, 31, 82, 245, 40, 103, 44, 208, 195, 220, 116, 150, 185, 6, 255, 56, 108, 232, 10, 105, 173, 242,
202, 36, 79, 97, 26, 117, 82, 13, 191, 105, 44, 174, 166, 252, 208, 230, 120, 131, 118, 193, 41, 100, 154, 136,
201, 96, 227, 7, 130, 203, 174, 160, 29, 220, 233, 148, 43, 230, 6, 86, 88, 85, 121, 71, 105, 49, 175, 139,
170, 59, 111, 108, 75, 22, 99, 42, 35, 250, 220, 0, 89, 18, 252, 102, 170, 142, 225, 44, 186, 45, 91, 37,
198, 210, 198, 245, 76, 60, 176, 223, 6, 103, 36, 17, 29, 243, 215, 45, 189, 203, 132, 221, 35, 57, 139, 203,
53, 223, 194, 35, 210, 173, 141, 27, 87, 115, 167, 236, 217, 93, 190, 99, 216, 149, 229, 220, 46, 114, 96, 78,
163, 138, 62, 133, 41, 15, 199, 190, 135, 36, 248, 196, 126, 127, 39, 100, 233, 223, 25, 181, 177, 227, 114, 41,
140, 238, 228, 99, 53, 17, 179, 200, 248, 22, 152, 126, 137, 190, 179, 182, 208, 179, 218, 39, 41, 151, 159, 216,
82, 159, 129, 31, 166, 17, 120, 200, 37, 191, 47, 244, 94, 132, 243, 1, 8, 197, 59, 80, 68, 170, 4, 167,
41, 15, 163, 250, 135, 178, 101, 235, 188, 160, 247, 97, 196, 91, 128, 107, 136, 26, 21, 64, 57, 75, 82, 117,
27, 70, 0, 100, 228, 94, 2, 213, 49, 253, 47, 249, 157, 51, 68, 90, 240, 25, 85, 194, 236, 11, 242, 210,
1, 208, 228, 99, 130, 210, 151, 198, 210, 36, 89, 196, 138, 147, 17, 180, 88, 97, 214, 142, 169, 11, 144, 214,
170, 210, 228, 189, 76, 103, 5, 254, 142, 250, 164, 215, 93, 229, 98, 26, 95, 110, 227, 2, 102, 254, 13, 183,
219, 72, 255, 240, 183, 158, 103, 225, 59, 96, 52, 159, 247, 160, 83, 187, 98, 125, 128, 199, 17, 38, 104, 211,
79, 37, 151, 96, 21, 254, 96, 19, 79, 254, 220, 213, 200, 86, 241, 120, 8, 149, 145, 248, 87, 132, 133, 86,
217, 192, 58, 166, 240, 91, 41, 9, 80, 209, 124, 149, 58, 31, 54, 79, 16, 83, 104, 191, 174, 235, 85, 152,
198, 209, 15, 174, 171, 164, 93, 138, 226, 164, 76, 119, 168, 46, 173, 194, 135, 70, 23, 34, 139, 60, 39, 244,
195, 103, 208, 233, 216, 250, 173, 141, 254, 129, 210, 125, 216, 97, 171, 112, 70, 127, 154, 15, 65, 74, 157, 136,
145, 34, 51, 134, 158, 121, 198, 241, 187, 149, 13, 250, 149, 129, 42, 152, 251, 239, 161, 140, 187, 203, 255, 167,
151, 136, 54, 221, 233, 209, 21, 77, 137, 206, 81, 174, 126, 124, 128, 64, 72, 60, 160, 65, 213, 131, 167, 38,
35, 19, 168, 22, 71, 188, 71, 75, 241, 195, 129, 10, 33, 238, 47, 153, 0, 167, 116, 125, 69, 129, 134, 23,
234, 108, 46, 140, 129, 46, 243, 83, 52, 241, 255, 70, 100, 116, 129, 72, 82, 242, 253, 246, 141, 42, 40, 162,
212, 225, 15, 72, 121, 58, 198, 214, 255, 116, 212, 241, 241, 83, 78, 110, 145, 88, 154, 57, 32, 106, 0, 62,
176, 169, 65, 168, 251, 244, 153, 80, 75, 238, 75, 203, 228, 152, 23, 21, 1, 172, 45, 64, 34, 218, 5, 163,
155, 248, 15, 106, 236, 111, 236, 166, 199, 82, 101, 142, 245, 36, 99, 100, 61, 23, 29, 190, 102, 106, 101, 162,
168, 93, 142, 111, 103, 129, 154, 252, 225, 62, 125, 226, 174, 2, 154, 173, 95, 9, 70, 192, 62, 231, 18, 142,
240, 153, 233, 128, 48, 168, 152, 33, 3, 225, 60, 7, 118, 217, 14, 17, 169, 65, 228, 97, 31, 97, 40, 149,
219, 157, 220, 66, 11, 82, 134, 81, 208, 197, 135, 136, 136, 127, 55, 196, 206, 230, 121, 162, 91, 134, 249, 1,
66, 21, 176, 113, 33, 182, 94, 145, 58, 61, 165, 213, 1, 141, 59, 40, 222, 199, 106, 23, 41, 132, 234, 191,
208, 38, 220, 125, 129, 155, 65, 170, 196, 204, 26, 115, 70, 45, 126, 97, 33, 193, 64, 30, 17, 208, 205, 49,
137, 108, 140, 215, 53, 11, 174, 62, 32, 40, 48, 161, 233, 215, 228, 46, 119, 41, 43, 252, 236, 152, 186, 141,
240, 4, 98, 255, 252, 109, 194, 173, 220, 127, 65, 37, 159, 208, 119, 197, 93, 112, 175, 166, 24, 8, 60, 50,
138, 75, 65, 28, 75, 214, 45, 173, 107, 8, 116, 54, 249, 141, 122, 136, 245, 206, 169, 59, 231, 215, 70, 115,
16, 146, 252, 239, 40, 93, 255, 75, 32, 47, 143, 204, 68, 198, 229, 21, 129, 25, 233, 92, 14, 219, 122, 5,
249, 154, 112, 58, 35, 3, 159, 42, 24, 186, 223, 163, 211, 16, 168, 134, 10, 81, 238, 209, 15, 142, 42, 160,
187, 74, 12, 251, 43, 80, 121, 94, 90, 155, 237, 189, 21, 55, 95, 170, 6, 39, 210, 249, 46, 37, 222, 231,
62, 43, 44, 95, 231, 206, 127, 225, 47, 157, 168, 32, 25, 213, 5, 102, 239, 46, 192, 150, 73, 157, 147, 54,
105, 210, 70, 30, 74, 24, 220, 1, 160, 249, 146, 9, 94, 242, 214, 134, 87, 95, 63, 134, 143, 64, 187, 244,
1, 211, 138, 93, 203, 16, 159, 143, 64, 53, 226, 14, 162, 195, 94, 41, 240, 252, 183, 58, 41, 206, 109, 82,
28, 107, 68, 155, 18, 48, 93, 253, 135, 123, 31, 208, 184, 191, 16, 51, 232, 242, 119, 125, 224, 24, 210, 162,
55, 201, 6, 100, 23, 156, 15, 106, 89, 56, 104, 33, 17, 82, 84, 44, 207, 30, 20, 216, 72, 124, 31, 149,
206, 76, 35, 205, 31, 144, 42, 25, 190, 96, 213, 143, 79, 255, 2, 46, 218, 54, 117, 26, 192, 204, 148, 110,
225, 61, 179, 34, 168, 91, 230, 238, 78, 78, 79, 249, 206, 196, 205, 85, 210, 144, 88, 89, 141, 255, 211, 205,
186, 122, 219, 167, 30, 149, 1, 176, 251, 151, 7, 72, 103, 244, 9, 204, 8, 94, 237, 23, 3, 233, 28, 215,
143, 83, 173, 196, 138, 235, 195, 191, 0, 19, 94, 195, 248, 223, 190, 115, 184, 79, 13, 1, 161, 172, 219, 37,
130, 27, 67, 196, 151, 124, 190, 251, 82, 99, 133, 15, 86, 90, 176, 58, 3, 246, 18, 184, 200, 101, 226, 254,
183, 142, 62, 129, 51, 159, 146, 113, 177, 68, 163, 27, 242, 2, 126, 183, 196, 8, 171, 227, 186, 67, 248, 243,
137, 60, 153, 168, 249, 60, 127, 106, 190, 231, 189, 222, 59, 35, 58, 92, 142, 177, 234, 28, 87, 143, 48, 172,
11, 65, 255, 66, 220, 191, 155, 90, 196, 139, 204, 72, 182, 151, 56, 167, 228, 65, 0, 40, 89, 86, 110, 13,
195, 252, 83, 157, 254, 98, 47, 148, 56, 152, 252, 93, 184, 190, 14, 53, 102, 30, 197, 56, 161, 156, 137, 198,
115, 146, 87, 147, 215, 216, 68, 103, 153, 183, 147, 123, 19, 185, 46, 233, 195, 80, 242, 79, 127, 102, 12, 238,
228, 226, 155, 180, 131, 86, 105, 251, 178, 84, 211, 244, 214, 118, 240, 44, 26, 15, 188, 22, 157, 44, 144, 127,
223, 95, 235, 19, 108, 15, 134, 7, 37, 6, 132, 16, 168, 216, 136, 184, 197, 30, 129, 16, 237, 131, 225, 141,
180, 121, 108, 211, 15, 54, 74, 73, 113, 185, 165, 243, 213, 148, 22, 2, 82, 39, 84, 39, 69, 0, 225, 169,
26, 58, 173, 27, 99, 127, 75, 143, 161, 245, 209, 26, 5, 61, 177, 26, 11, 26, 241, 169, 140, 90, 16, 220,
137, 185, 244, 143, 105, 174, 97, 205, 237, 253, 185, 133, 160, 107, 143, 12, 164, 14, 142, 72, 218, 212, 33, 58,
210, 43, 244, 235, 132, 114, 127, 83, 177, 12, 132, 171, 45, 230, 14, 118, 119, 201, 251, 227, 230, 91, 144, 220,
25, 97, 30, 81, 67, 222, 128, 172, 6, 229, 63, 115, 201, 57, 216, 136, 196, 164, 31, 182, 70, 190, 70, 38,
254, 179, 146, 147, 231, 27, 130, 137, 190, 243, 199, 136, 138, 51, 221, 237, 196, 151, 108, 133, 110, 112, 223, 1,
7, 222, 213, 135, 12, 157, 158, 140, 235, 115, 221, 197, 65, 35, 143, 246, 48, 98, 6, 100, 123, 183, 121, 170,
52, 175, 22, 96, 19, 118, 64, 58, 225, 100, 137, 213, 213, 58, 44, 202, 205, 196, 205, 241, 173, 214, 218, 163,
52, 50, 106, 255, 176, 69, 176, 133, 192, 68, 176, 178, 15, 194, 63, 23, 44, 3, 61, 243, 136, 197, 139, 231,
219, 220, 32, 48, 184, 175, 26, 1, 150, 125, 149, 255, 249, 40, 40, 118, 135, 169, 41, 108, 222, 4, 14, 198,
51, 92, 219, 216, 123, 64, 123, 167, 187, 158, 87, 7, 121, 212, 200, 20, 3, 90, 50, 105, 79, 155, 95, 188,
18, 67, 127, 222, 251, 49, 147, 172, 165, 236, 232, 134, 26, 7, 226, 225, 194, 74, 79, 247, 196, 199, 226, 97,
80, 244, 117, 228, 158, 53, 10, 13, 19, 72, 200, 65, 254, 48, 95, 179, 115, 70, 122, 0, 22, 113, 140, 196,
107, 128, 157, 152, 78, 157, 30, 85, 105, 78, 3, 183, 73, 20, 124, 122, 192, 161, 74, 207, 232, 48, 73, 80,
179, 169, 164, 89, 21, 153, 106, 8, 27, 67, 148, 36, 41, 152, 195, 171, 30, 227, 56, 23, 230, 109, 160, 20,
226, 195, 254, 226, 196, 103, 52, 167, 200, 90, 214, 139, 97, 106, 161, 218, 173, 3, 86, 61, 123, 222, 0, 51,
59, 20, 178, 73, 52, 155, 114, 69, 142, 140, 9, 225, 120, 153, 39, 124, 215, 201, 118, 47, 224, 23, 59, 161,
105, 60, 172, 105, 13, 210, 11, 219, 21, 100, 231, 72, 121, 19, 142, 184, 4, 228, 89, 104, 242, 77, 74, 27,
215, 211, 29, 237, 216, 55, 25, 60, 63, 174, 128, 137, 0, 40, 130, 183, 189, 250, 90, 173, 149, 162, 136, 25,
64, 26, 73, 62, 77, 74, 255, 10, 8, 55, 204, 131, 202, 91, 79, 3, 68, 154, 16, 246, 202, 239, 138, 67,
112, 158, 94, 172, 203, 198, 150, 45, 1, 42, 190, 171, 255, 149, 141, 41, 123, 149, 63, 182, 55, 3, 180, 48,
209, 57, 115, 31, 188, 173, 190, 193, 34, 211, 63, 2, 79, 175, 248, 14, 39, 218, 104, 117, 238, 4, 110, 90,
20, 213, 187, 87, 11, 35, 45, 77, 42, 40, 12, 60, 72, 84, 146, 11, 69, 195, 124, 148, 244, 56, 82, 215,
122, 163, 57, 247, 251, 218, 245, 124, 136, 220, 190, 171, 97, 127, 193, 215, 90, 221, 222, 165, 71, 194, 251, 243,
224, 75, 66, 101, 39, 122, 220, 160, 243, 11, 172, 163, 166, 148,
]
type
# The value's UTF-8 bytes in the low bytes and its length in the top byte.
Entry = object
key, val: uint64
Long = object
name: array[32, uint8]
val: uint64
len: uint32
pad: uint32
V128 {.importc: "v128_t", header: "", bycopy.} = object
Bytes = ptr UncheckedArray[uint8]
proc vload(p: pointer): V128 {.importc: "wasm_v128_load", header: "".}
proc vstore(p: pointer, v: V128) {.importc: "wasm_v128_store", header: "".}
proc splat(x: int8): V128 {.importc: "wasm_i8x16_splat", header: "".}
proc eq8(a, b: V128): V128 {.importc: "wasm_i8x16_eq", header: "".}
proc bitmask(a: V128): uint32 {.importc: "wasm_i8x16_bitmask", header: "".}
proc vand(a, b: V128): V128 {.importc: "wasm_v128_and", header: "".}
proc vor(a, b: V128): V128 {.importc: "wasm_v128_or", header: "".}
proc vxor(a, b: V128): V128 {.importc: "wasm_v128_xor", header: "".}
proc anyTrue(a: V128): bool {.importc: "wasm_v128_any_true", header: "".}
proc ctz64(x: uint64): cint {.importc: "__builtin_ctzll", nodecl.}
proc ctz32(x: uint32): cint {.importc: "__builtin_ctz", nodecl.}
proc popcount64(x: uint64): cint {.importc: "__builtin_popcountll", nodecl.}
proc fill(p: pointer, c: cint, n: csize_t) {.importc: "__builtin_memset", nodecl.}
proc copy(d, s: pointer, n: csize_t) {.importc: "__builtin_memcpy", nodecl.}
var
# One buffer: output from its start, input InOff bytes in. On the first call its start
# also holds the unpacker's model, which is why the model has 2^17 16-bit entries.
buf {.align: 64.}: array[InOff + InputCap + Pad, uint8]
fast: array[(1 shl FastBits) + FastTail, Entry]
leg: array[(1 shl LegBits) + LegTail, Entry]
longs: array[LongCount, Long]
longSlots: array[1 shl LongBits, uint16]
# Longest legacy name starting with these two bytes (hashed), or 0.
legMax: array[4096, uint8]
pos: array[Window + 16, uint32]
# Byte j of the 32 loaded at lenMask + 32 - n is 0xFF when j < n.
lenMask: array[64, uint8]
ready: bool
const
# Windows-1252 remapping for references to 0x80-0x9F.
c1: array[32, uint16] = [
0x20AC'u16, 0x81, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x8D, 0x017D, 0x8F,
0x90, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178]
utf8Marks: array[5, uint32] = [0'u32, 0, 0x80C0, 0x8080E0, 0x808080F0'u32]
template input: Bytes = cast[Bytes](addr buf[InOff])
template output: Bytes = cast[Bytes](addr buf[0])
template ld64(p: pointer): uint64 = cast[ptr uint64](p)[]
template st64(p: pointer, x: uint64) = cast[ptr uint64](p)[] = x
template st32(p: pointer, x: uint32) = cast[ptr uint32](p)[] = x
template at(p: Bytes, i: uint32): pointer = cast[pointer](addr p[i])
# A constant offset is applied to the pointer, not the index, so that it folds into the
# load or store instruction.
template at(p: Bytes, i: uint32, k: int): pointer = cast[pointer](addr cast[Bytes](addr p[i])[k])
template legSlot(w: uint64): uint32 = ((uint32(w and 0xFFFF)) * 0x9E3779B1'u32) shr 20
template rotl(x: uint64, r: int): uint64 = (x shl r) or (x shr (64 - r))
template longHash(head, tail: uint64, n: uint32): uint32 =
uint32((((head xor rotl(tail, 29)) + n) * LongMul) shr (64 - LongBits))
# Lay the bits of a code point out as four UTF-8 bytes, then drop the unused leading ones.
template utf8(v: uint32, len: var uint32): uint32 =
let n = 1'u32 + uint32(v > 0x7F) + uint32(v > 0x7FF) + uint32(v > 0xFFFF)
let bits = (v shr 18) or ((v shr 12 and 0x3F) shl 8) or ((v shr 6 and 0x3F) shl 16) or ((v and 0x3F) shl 24)
len = n
if n == 1: v else: (bits shr (32 - 8 * n)) or utf8Marks[n]
# The entity list is stored packed and read once, on the first call. Entities are ordered
# by code point, so a value is a small step from the one before. A name is coded character
# by character with an adaptive binary range coder (the LZMA one) from six contexts: the
# previous one, two and three characters; a match model that predicts the character that
# followed the last place the previous two were seen; and two that look at the same
# position in the previous entity's name, since neighbours in code point order are often
# named alike (Afr, Bfr; Acy, Bcy; Aacute, Acirc).
#
# Each context has a model entry of 16 bits: a 12-bit probability (stored XOR 0x800, so
# zeroed memory means one half) above a 4-bit count of its uses. The entries' probabilities
# are combined by logistic mixing, as in the PAQ compressors:
# p = squash(sum of weight * stretch(p_i)), stretch(p) = ln(p / (1 - p)), and after the bit
# is known each weight moves by error * stretch(p_i).
var
squash: array[4095, uint16]
stretch: array[4096, int16]
# Five sets of six weights, 16.16 fixed point, stored less MixW0.
weights: array[5, array[6, int32]]
ctx: array[6, uint32]
hist: array[HistSize, uint8]
last: array[4096, uint16]
rcIn, rcRange, rcCode: uint32
template model: ptr UncheckedArray[uint16] = cast[ptr UncheckedArray[uint16]](addr buf[0])
proc utf8Once(v: uint32, len: var uint32): uint32 {.noinline.} = utf8(v, len)
proc sym(ws, nctx, nbits: uint32): uint32 {.noinline.} =
var node = 1'u32
let top = 1'u32 shl nbits
let w = addr weights[ws]
while true:
var slot {.noinit.}: array[6, uint32]
var st {.noinit.}: array[6, int32]
var dot = 0'i32
for k in 0'u32 ..< nctx:
let s = ((ctx[k] + node * 0x2545F491'u32) * 0x9E3779B1'u32) shr (32 - ModelBits)
slot[k] = s
st[k] = int32(stretch[(model[s] shr 4) xor 0x800])
dot += st[k] * (w[k] + MixW0)
dot = ashr(dot, 16) + 2047
if dot < 0: dot = 0
if dot > 4094: dot = 4094
# 12-bit probability that the bit is 1, from 1 to 4095.
let p = uint32(squash[dot])
let bound = (rcRange shr 12) * p
let bit = uint32(rcCode < bound)
if bit != 0:
rcRange = bound
else:
rcCode -= bound
rcRange -= bound
while rcRange < (1'u32 shl 24):
rcRange = rcRange shl 8
rcCode = (rcCode shl 8) or packed[rcIn]
rcIn += 1
# Move each weight by error * input, and each entry toward the bit just seen by
# 2 / (2n + 3) after n uses.
let one = int32(bit shl 12)
let err = one - int32(p)
for k in 0'u32 ..< nctx:
w[k] += ashr(st[k] * err, MixShift)
let e = uint32(model[slot[k]])
var q = int32((e shr 4) xor 0x800)
let n = e and 15
q += ashr((one - q) * int32(rate[n]), 16)
model[slot[k]] = uint16(((uint32(q) xor 0x800) shl 4) or (n + uint32(n < ModelLimit)))
node = node * 2 + bit
if node >= top: break
node - top
proc init() {.noinline.} =
fill(addr fast, 255, csize_t(sizeof(fast)))
fill(addr leg, 255, csize_t(sizeof(leg)))
fill(addr lenMask, 255, 32)
# squash[x + 2047] = 4096 / (1 + e^(-x / 256)) rounded, for x = -2047..2047, and stretch,
# its inverse, filled in as squash rises. Only IEEE multiply, add and divide are used,
# so the tables are the same bits the packer had.
var e = cast[float64](SquashE0)
var pi = 0'u32
for x in 0'u32 ..< 4095:
let v = (uint32(8192.0 / (1.0 + e)) + 1) shr 1
squash[x] = uint16(v)
while pi <= v:
stretch[pi] = int16(int32(x) - 2047)
pi += 1
e = e * cast[float64](SquashStep)
rcIn = 4
rcRange = not 0'u32
rcCode = (uint32(packed[0]) shl 24) or (uint32(packed[1]) shl 16) or (uint32(packed[2]) shl 8) or packed[3]
var cp, nl, start = 0'u32
var hn = 1'u32
for r in 0 ..< EntityCount:
# Step from the previous code point: its bit length, then the bits below the top one.
ctx[0] = K
let nb = sym(4, 1, 5)
var d = nb
if nb > 1:
ctx[0] = 2 * K + nb
d = (1'u32 shl (nb - 1)) + sym(4, 1, nb - 1)
cp += d
var vn, vn2 = 0'u32
var val = uint64(utf8Once(cp, vn))
# A few values have a second code point.
ctx[0] = 3 * K + uint32(d == 0)
if sym(4, 1, 1) != 0:
ctx[0] = 4 * K
val = val or (uint64(utf8Once(sym(4, 1, 16), vn2)) shl (8 * vn))
val = val or (uint64(vn + vn2) shl 56)
ctx[0] = 5 * K + uint32(cp < 256)
let legacy = sym(4, 1, 1)
# Names use a 6-bit alphabet: 0 ends the name, then digits, capitals, small letters.
var name {.noinit.}: array[40, uint8]
fill(addr name, 0, 40)
var n, h, mp, ml = 0'u32
let prev = start
start = hn
while true:
let m3 = min(ml, 3)
# The character at this position in the previous entity's name (or, past its end,
# whatever follows it in hist).
let pc = uint32(hist[prev + n]) shl 6
ctx[0] = 8 * K + (h and 63)
ctx[1] = 9 * K + (h and 4095)
ctx[2] = 13 * K + (if ml != 0: uint32(hist[mp]) else: 0'u32) * 16 + m3
ctx[3] = 10 * K + (h and 0x3FFFF)
ctx[4] = 14 * K + pc + (h and 63)
ctx[5] = 17 * K + pc + min(n, 3) * 4 + min(d, 3)
# One set of weights for each state of the match model; set 4 is for the fields above.
let c = sym(m3, 6, 6)
if ml != 0 and hist[mp] == c:
ml += 1
mp += 1
else:
ml = 0
hist[hn] = uint8(c)
hn += 1
if c == 0: break
name[n] = uint8(c + (if c < 11: 47'u32 elif c < 37: 54'u32 else: 60'u32))
n += 1
h = (h shl 6) or c
if n >= 2:
let l = addr last[h and 4095]
if ml == 0 and l[] != 0:
mp = l[]
ml = 1
l[] = uint16(hn)
if n <= 7:
# fast: the key is the name and its `;`.
var key = ld64(addr name) or (59'u64 shl (8 * n))
var e = uint32((key * FastMul) shr (64 - FastBits))
while fast[e].key != Empty: e += 1
fast[e].key = key
fast[e].val = val
if legacy != 0:
# leg: the length is part of the key, so a NUL after a shorter name cannot match.
key = ld64(addr name) or (uint64(n) shl 56)
e = uint32((key * LegMul) shr (64 - LegBits))
while leg[e].key != Empty: e += 1
leg[e].key = key
leg[e].val = val
let s = legSlot(key) # only the first two bytes matter
if legMax[s] < n: legMax[s] = uint8(n)
else:
copy(addr longs[nl].name, addr name, 32)
longs[nl].val = val
longs[nl].len = n
nl += 1
var hh = longHash(ld64(addr name), ld64(addr name[n - 8]), n)
while longSlots[hh] != 0: hh = (hh + 1) and ((1 shl LongBits) - 1)
longSlots[hh] = uint16(nl)
ready = true
proc input_ptr(): uint32 {.exportc.} = cast[uint32](addr buf[InOff])
proc input_utf8_cap(): uint32 {.exportc.} = InputCap
proc output_utf8_cap(): uint32 {.exportc.} = OutputCap
proc render(n: uint32): uint64 {.exportc.} =
if unlikely(not ready): init()
let size = min(n, InputCap)
let src = input
let dst = output
let zero = splat(0)
vstore(src.at(size), zero)
vstore(src.at(size, 16), zero)
vstore(src.at(size, 32), zero)
vstore(src.at(size, 48), zero)
# Stage 1 lists the position of every `&` in a window; stage 2 copies the text between
# references and decodes each one. Knowing the next position before the current
# reference is decoded keeps the references independent of each other.
var e, o, base = 0'u32
let amp = splat(38)
let semi = splat(59)
while base < size:
let lim = if size - base > Window: base + Window else: size
var count = 0'u32
while true:
var mm = uint64(bitmask(eq8(vload(src.at(base)), amp))) or
(uint64(bitmask(eq8(vload(src.at(base, 16)), amp))) shl 16) or
(uint64(bitmask(eq8(vload(src.at(base, 32)), amp))) shl 32) or
(uint64(bitmask(eq8(vload(src.at(base, 48)), amp))) shl 48)
let cnt = uint32(popcount64(mm))
let outp = cast[ptr UncheckedArray[uint32]](addr pos[count])
# Entries past cnt are garbage and are overwritten or ignored.
outp[0] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
outp[1] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
outp[2] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
outp[3] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
if unlikely(cnt > 4):
outp[4] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
outp[5] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
outp[6] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
outp[7] = base + uint32(ctz64(mm)); mm = mm and (mm - 1)
var k = 8'u32
while unlikely(mm != 0):
outp[k] = base + uint32(ctz64(mm))
mm = mm and (mm - 1)
k += 1
count += cnt
base += 64
if base >= lim: break
for q in 0'u32 ..< count:
var i = pos[q]
# Output and input are one buffer, so the compiler cannot move a load across a store.
# Everything this reference needs is therefore loaded before the first store.
let w = ld64(src.at(i, 1))
# Copy the text since the last reference; src[i] is `&` and i < size.
block:
let len = i - e
let ca = vload(src.at(e))
let cb = vload(src.at(e, 16))
vstore(dst.at(o), ca)
vstore(dst.at(o, 16), cb)
if len > 32:
var j = 32'u32
while true:
vstore(dst.at(o + j), vload(src.at(e + j)))
j += 16
if j >= len: break
o += len
block done:
block bare:
if uint8(w and 0xFF) == 35:
# Decimal and hexadecimal share one branch-free path: classify 8 bytes, take
# the digits before the first other byte, and fold them pairwise.
let hex = ((uint32(w shr 8) and 0xFF) or 0x20) == 120
var p = i + 2 + uint32(hex)
let d = ld64(src.at(p))
let t = d xor 0x3030303030303030'u64
let digit = not (((t and not High) + 0x7676767676767676'u64) or t) and High
let y = d or 0x2020202020202020'u64
let ty = y and not High
let letter = (ty + Ones * (0x80 - 97)) and not (ty + Ones * (0x7F - 102)) and not y and High
let stop = not (digit or (if hex: letter else: 0'u64)) and High
var v, last: uint32
if likely(stop != 0):
let nb = uint32(ctz64(stop)) and 0x38
if unlikely(nb == 0): break bare
var x = ((d and 0x0F0F0F0F0F0F0F0F'u64) + 9 * (letter shr 7)) shl (64 - nb)
x = (x * (if hex: 16'u64 else: 10'u64) + (x shr 8)) and 0x00FF00FF00FF00FF'u64
x = (x * (if hex: 256'u64 else: 100'u64) + (x shr 16)) and 0x0000FFFF0000FFFF'u64
v = uint32((x * (if hex: 65536'u64 else: 10000'u64) + (x shr 32)) and 0xFFFFFFFF'u64)
last = p + (nb shr 3)
else:
# Eight or more digits: saturate past U+10FFFF.
let radix = if hex: 16'u32 else: 10'u32
v = 0
while true:
var c = uint32(src[p]) - 48
if c > 9:
c = ((uint32(src[p]) or 0x20) - 97) + 10
if c < 10: c = 16
if c >= radix: break
v = v * radix + c
if v > 0x10FFFF: v = 0x110000
p += 1
last = p
i = last + uint32(src[last] == 59)
if unlikely(v - 0x80 < 0x20): v = c1[v - 0x80]
elif unlikely(v == 0 or v - 0xD800 < 0x800 or v > 0x10FFFF): v = 0xFFFD
var len: uint32
let word = utf8(v, len)
st32(dst.at(o), word)
o += len
break done
let t = w xor (Ones * 59)
let z = (t - Ones) and not t and High
if likely(z != 0):
# Bit index of the `;` byte; the key is the name and its `;`.
let kb = uint32(ctz64(z)) and 0x38
let key = w and ((2'u64 shl (kb + 7)) - 1)
# Each probe loop only finds the slot; acting on it afterwards keeps the loop
# free of exits into the code after the reference.
var s = uint32((key * FastMul) shr (64 - FastBits))
var k = fast[s].key
while unlikely(k != key and k != Empty):
s += 1
k = fast[s].key
if likely(k == key):
let val = fast[s].val
st64(dst.at(o), val)
o += uint32(val shr 56)
i += 2 + (kb shr 3)
break done
else:
# No `;` in 8 bytes: a name of 8 to 31 bytes needs its `;` in the next 24.
let s0 = vload(src.at(i, 1))
let s1 = vload(src.at(i, 17))
let sm = bitmask(eq8(s0, semi)) or (bitmask(eq8(s1, semi)) shl 16)
if sm != 0:
let len = uint32(ctz32(sm))
var h = longHash(w, ld64(src.at(i + len, -7)), len)
let m0 = vload(addr lenMask[32 - len])
let m1 = vload(addr lenMask[48 - len])
var found = 0'u64
while true:
let s = longSlots[h]
if s == 0: break
let l = addr longs[s - 1]
let diff = vor(vxor(vand(s0, m0), vload(addr l.name)), vxor(vand(s1, m1), vload(addr l.name[16])))
if l.len == len and not anyTrue(diff):
found = l.val
break
h = (h + 1) and ((1 shl LongBits) - 1)
if found != 0:
st64(dst.at(o), found)
o += uint32(found shr 56)
i += 2 + len
break done
# Without `;`, the longest legacy name that is a prefix.
var k = uint32(legMax[legSlot(w)])
var found = 0'u64
while k >= 2:
# The length is part of the key, so a NUL after a shorter name cannot match.
let key = (w and ((1'u64 shl (8 * k)) - 1)) or (uint64(k) shl 56)
var s = uint32((key * LegMul) shr (64 - LegBits))
while leg[s].key != key and leg[s].key != Empty: s += 1
if leg[s].key == key:
found = leg[s].val
break
k -= 1
if found != 0:
st64(dst.at(o), found)
o += uint32(found shr 56)
i += 1 + k
break done
# A bare `&`.
dst[o] = 38
o += 1
i += 1
e = i
# Without a reference the input is the output.
if e == 0: return (uint64(cast[uint32](addr buf[InOff])) shl 32) or size
var j = e
while j < size:
vstore(dst.at(o + (j - e)), vload(src.at(j)))
j += 16
o += size - e
(uint64(cast[uint32](addr buf[0])) shl 32) or o
{.pop.}
#[ Build (build-nim.sh):
#!/bin/sh
# Generates unescape.nim (the single-file source to submit) from unescape.in.nim and builds
# unescape-nim.wasm: Nim 2.4.0 translates to C, zig cc (clang) compiles that to Wasm.
set -eu
cd "$(dirname "$0")"
python3 gen-small.py unescape.in.nim unescape.nim 17
NIM="mise exec nim@2.4.0 --"
# --os:standalone asks for this file. Checks are off, so neither proc is ever called.
cat > panicoverride.nim <<'PANIC'
proc rawoutput(s: string) = discard
proc panic(s: string) {.noreturn.} =
while true: discard
PANIC
C="${TMPDIR:-/tmp}/unescape-nimcache"
rm -rf "$C"
$NIM nim c --compileOnly --nimcache:"$C" --cpu:wasm32 --os:standalone --mm:none --threads:off \
-d:danger -d:noSignalHandler --noMain --app:staticlib unescape.nim
LIB="$(dirname "$($NIM which nim)")/../lib"
zig cc -target wasm32-freestanding -O3 -msimd128 -mbulk-memory -nostdlib -fno-strict-aliasing -I"$LIB" \
-Wl,--no-entry -Wl,--export-memory -Wl,--export=input_ptr -Wl,--export=input_utf8_cap \
-Wl,--export=output_utf8_cap -Wl,--export=render \
-Wl,--initial-memory=1507328 -Wl,--max-memory=1507328 -Wl,-z,stack-size=4096 -Wl,--strip-all \
-o unescape-nim.wasm "$C"/*.c
ls -l unescape-nim.wasm
pack_mix.py, which produced `packed`:
"""Packs the entity list for unescape.wat: entities ordered by code point, coded with an
adaptive binary range coder. Each bit is predicted from several contexts whose predictions
are combined by a small learned mixer (logistic mixing, as in the PAQ compressors).
$sym and $init in unescape.wat mirror Model, Decoder and Coder exactly.
"""
import html.entities
M32 = 0xFFFFFFFF
K = 1 << 20
HASH_A, HASH_B = 0x2545F491, 0x9E3779B1
# Step toward the bit just seen, as a 16-bit fraction: 2 / (2n + 3) after n uses.
RATE = [43690, 26214, 18724, 14563]
# e^(2047 / 256) and e^(-1 / 256), written out so that every platform uses the same bits.
E0, STEP = float.fromhex('0x1.732ac33de5528p+11'), float.fromhex('0x1.fe00ffaabffbcp-1')
html5 = html.entities.html5
NAMES = {k[:-1]: v for k, v in html5.items() if k.endswith(';')}
LEGACY = {k for k in html5 if not k.endswith(';')}
ALPHA = '\0' + '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
CODE = {c: i for i, c in enumerate(ALPHA)}
ORDER = sorted(NAMES, key=lambda n: ([ord(c) for c in NAMES[n]], n))
def tables():
"""squash[x + 2047] = 4096 / (1 + e^(-x / 256)) rounded, for x = -2047..2047, and
stretch, its inverse. Only IEEE multiply, add and divide, so every machine agrees."""
squash, stretch, e, pi = [], [0] * 4096, E0, 0
for x in range(-2047, 2048):
v = (int(8192 / (1 + e)) + 1) >> 1
squash.append(v)
while pi <= v:
stretch[pi] = x
pi += 1
e *= STEP
assert pi == 4096 and squash[0] >= 1
return squash, stretch
class Model:
"""Entries are 16 bits: a 12-bit probability (stored XOR 0x800) and a 4-bit count.
Weight set 4 is for the code point fields; 0 to 3 are for name characters, chosen
by how long the match model has been right."""
def __init__(self, bits=17, limit=3, shift=11, w0=16384, sets=5, nmax=6):
self.t = [0] * (1 << bits)
self.bits, self.limit, self.shift, self.w0 = bits, limit, shift, w0
self.squash, self.stretch = tables()
self.w = [[0] * nmax for _ in range(sets)]
def slots(self, ctxs, node):
return [(((c + node * HASH_A) * HASH_B) & M32) >> (32 - self.bits) for c in ctxs]
def predict(self, slots, ws):
"""12-bit probability that the bit is 1, and the inputs it was mixed from."""
st = [self.stretch[(self.t[s] >> 4) ^ 0x800] for s in slots]
dot = sum((w + self.w0) * x for w, x in zip(self.w[ws], st))
assert abs(dot) < 1 << 31 # the unpacker adds these up in 32 bits
dot >>= 16
return self.squash[min(max(dot, -2047), 2047) + 2047], st
def update(self, slots, ws, st, p, bit):
err = (bit << 12) - p
w = self.w[ws]
for k, s in enumerate(slots):
w[k] += (st[k] * err) >> self.shift
assert abs(w[k]) < 1 << 30
e = self.t[s]
q, n = (e >> 4) ^ 0x800, e & 15
q += (((bit << 12) - q) * RATE[n]) >> 16
self.t[s] = (q ^ 0x800) << 4 | (n + (n < self.limit))
class Encoder:
def __init__(self):
self.low, self.range, self.cache, self.pending, self.out = 0, M32, 0, 1, bytearray()
def shift_low(self):
if self.low < 0xFF000000 or self.low > M32:
carry = self.low >> 32
byte = self.cache
while self.pending:
self.out.append((byte + carry) & 0xFF)
byte = 0xFF
self.pending -= 1
self.cache = self.low >> 24 & 0xFF
self.pending += 1
self.low = (self.low << 8) & M32
def bit(self, p, bit):
bound = (self.range >> 12) * p
if bit:
self.range = bound
else:
self.low += bound
self.range -= bound
while self.range < 1 << 24:
self.range = (self.range << 8) & M32
self.shift_low()
def finish(self):
for _ in range(5):
self.shift_low()
assert self.out[0] == 0
return bytes(self.out[1:])
class Decoder:
def __init__(self, data):
self.data, self.pos, self.range = data + bytes(8), 4, M32
self.code = int.from_bytes(data[:4], 'big')
def bit(self, p):
bound = (self.range >> 12) * p
if self.code < bound:
self.range = bound
bit = 1
else:
self.code -= bound
self.range -= bound
bit = 0
while self.range < 1 << 24:
self.range = (self.range << 8) & M32
self.code = (self.code << 8 | self.data[self.pos]) & M32
self.pos += 1
return bit
class Coder:
"""Runs the same field sequence for packing (values given) and unpacking (values None)."""
def __init__(self, data=None, **kw):
self.m = Model(**kw)
self.enc = Encoder() if data is None else None
self.dec = Decoder(data) if data is not None else None
self.hist, self.last, self.start = [0], [0] * 4096, 0
def sym(self, ctxs, nbits, ws, value=None):
node = 1
for i in range(nbits - 1, -1, -1):
slots = self.m.slots(ctxs, node)
p, st = self.m.predict(slots, ws)
if self.enc:
b = value >> i & 1
self.enc.bit(p, b)
else:
b = self.dec.bit(p)
self.m.update(slots, ws, st, p, b)
node = node * 2 + b
return node - (1 << nbits)
def entity(self, cp, rec=None):
"""rec = (name, [code points], legacy) when packing. Returns the same when unpacking."""
name, cps, legacy = rec if rec else (None, None, None)
d = cps[0] - cp if rec else None
nb = self.sym([1 * K], 5, 4, d.bit_length() if rec else None)
if nb > 1:
d = (1 << (nb - 1)) + self.sym([2 * K + nb], nb - 1, 4, d - (1 << (nb - 1)) if rec else None)
else:
d = nb
cp += d
second = 0
if self.sym([3 * K + (d == 0)], 1, 4, len(cps) - 1 if rec else None):
second = self.sym([4 * K], 16, 4, cps[1] if rec else None)
leg = self.sym([5 * K + (cp < 256)], 1, 4, int(legacy) if rec else None)
# The match model predicts the character that followed the last place the previous
# two characters were seen, and how many predictions in a row have been right.
# prev walks the previous entity's name in step with this one.
h, out, mp, ml, prev = 0, '', 0, 0, self.start
self.start = len(self.hist)
for i in range(64):
pc = self.hist[prev + i]
ctxs = [8 * K + (h & 63), 9 * K + (h & 4095),
13 * K + (self.hist[mp] if ml else 0) * 16 + min(ml, 3), 10 * K + (h & 0x3FFFF),
14 * K + pc * 64 + (h & 63), 17 * K + pc * 64 + min(i, 3) * 4 + min(d, 3)]
c = self.sym(ctxs, 6, min(ml, 3), CODE[(name + '\0')[i]] if rec else None)
if ml and self.hist[mp] == c:
ml += 1
mp += 1
else:
ml = 0
self.hist.append(c)
if c == 0:
break
out += ALPHA[c]
h = h << 6 | c
if i >= 1:
if ml == 0 and self.last[h & 4095]:
mp, ml = self.last[h & 4095], 1
self.last[h & 4095] = len(self.hist)
return out, [cp, second] if second else [cp], bool(leg)
def records():
return [(n, [ord(c) for c in NAMES[n]], n in LEGACY) for n in ORDER]
def pack(**kw):
c = Coder(**kw)
cp = 0
for rec in records():
assert len(rec[1]) <= 2 and rec[1][0] >= cp and (len(rec[1]) == 1 or rec[1][1] < 0x10000)
c.entity(cp, rec)
cp = rec[1][0]
return c.enc.finish()
def unpack(data, count, **kw):
c = Coder(data, **kw)
cp, out = 0, []
for _ in range(count):
rec = c.entity(cp)
cp = rec[1][0]
out.append(rec)
return out
if __name__ == '__main__':
blob = pack()
assert unpack(blob, len(ORDER)) == records()
print('round trip ok', len(blob))
]#