1 /* enough.c -- determine the maximum size of inflate's Huffman code tables over
2 * all possible valid and complete prefix codes, subject to a length limit.
3 * Copyright (C) 2007, 2008, 2012, 2018 Mark Adler
4 * Version 1.5 1 August 2018 Mark Adler
8 1.0 3 Jan 2007 First version (derived from codecount.c version 1.4)
9 1.1 4 Jan 2007 Use faster incremental table usage computation
10 Prune examine() search on previously visited states
11 1.2 5 Jan 2007 Comments clean up
12 As inflate does, decrease root for short codes
13 Refuse cases where inflate would increase root
14 1.3 17 Feb 2008 Add argument for initial root table size
15 Fix bug for initial root table size == max - 1
16 Use a macro to compute the history index
17 1.4 18 Aug 2012 Avoid shifts more than bits in type (caused endless loop!)
18 Clean up comparisons of different types
19 Clean up code indentation
20 1.5 1 Aug 2018 Clean up code style, formatting, and comments
21 Use inline function instead of macro for index
25 Examine all possible prefix codes for a given number of symbols and a
26 maximum code length in bits to determine the maximum table size for zlib's
27 inflate. Only complete prefix codes are counted.
29 Two codes are considered distinct if the vectors of the number of codes per
30 length are not identical. So permutations of the symbol assignments result
31 in the same code for the counting, as do permutations of the assignments of
32 the bit values to the codes (i.e. only canonical codes are counted).
34 We build a code from shorter to longer lengths, determining how many symbols
35 are coded at each length. At each step, we have how many symbols remain to
36 be coded, what the last code length used was, and how many bit patterns of
37 that length remain unused. Then we add one to the code length and double the
38 number of unused patterns to graduate to the next code length. We then
39 assign all portions of the remaining symbols to that code length that
40 preserve the properties of a correct and eventually complete code. Those
41 properties are: we cannot use more bit patterns than are available; and when
42 all the symbols are used, there are exactly zero possible bit patterns
45 The inflate Huffman decoding algorithm uses two-level lookup tables for
46 speed. There is a single first-level table to decode codes up to root bits
47 in length (root == 9 in the current inflate implementation). The table has 1
48 << root entries and is indexed by the next root bits of input. Codes shorter
49 than root bits have replicated table entries, so that the correct entry is
50 pointed to regardless of the bits that follow the short code. If the code is
51 longer than root bits, then the table entry points to a second- level table.
52 The size of that table is determined by the longest code with that root-bit
53 prefix. If that longest code has length len, then the table has size 1 <<
54 (len - root), to index the remaining bits in that set of codes. Each
55 subsequent root-bit prefix then has its own sub-table. The total number of
56 table entries required by the code is calculated incrementally as the number
57 of codes at each bit length is populated. When all of the codes are shorter
58 than root bits, then root is reduced to the longest code length, resulting
59 in a single, smaller, one-level table.
61 The inflate algorithm also provides for small values of root (relative to
62 the log2 of the number of symbols), where the shortest code has more bits
63 than root. In that case, root is increased to the length of the shortest
64 code. This program, by design, does not handle that case, so it is verified
65 that the number of symbols is less than 2^(root + 1).
67 In order to speed up the examination (by about ten orders of magnitude for
68 the default arguments), the intermediate states in the build-up of a code
69 are remembered and previously visited branches are pruned. The memory
70 required for this will increase rapidly with the total number of symbols and
71 the maximum code length in bits. However this is a very small price to pay
74 First, all of the possible prefix codes are counted, and reachable
75 intermediate states are noted by a non-zero count in a saved-results array.
76 Second, the intermediate states that lead to (root + 1) bit or longer codes
77 are used to look at all sub-codes from those junctures for their inflate
78 memory usage. (The amount of memory used is not affected by the number of
79 codes of root bits or less in length.) Third, the visited states in the
80 construction of those sub-codes and the associated calculation of the table
81 size is recalled in order to avoid recalculating from the same juncture.
82 Beginning the code examination at (root + 1) bit codes, which is enabled by
83 identifying the reachable nodes, accounts for about six of the orders of
84 magnitude of improvement for the default arguments. About another four
85 orders of magnitude come from not revisiting previous states. Out of
86 approximately 2x10^16 possible prefix codes, only about 2x10^6 sub-codes
87 need to be examined to cover all of the possible table memory usage cases
88 for the default arguments of 286 symbols limited to 15-bit codes.
90 Note that an unsigned long long type is used for counting. It is quite easy
91 to exceed the capacity of an eight-byte integer with a large number of
92 symbols and a large maximum code length, so multiple-precision arithmetic
93 would need to replace the unsigned long long arithmetic in that case. This
94 program will abort if an overflow occurs. The big_t type identifies where
95 the counting takes place.
97 An unsigned long long type is also used for calculating the number of
98 possible codes remaining at the maximum length. This limits the maximum code
99 length to the number of bits in a long long minus the number of bits needed
100 to represent the symbols in a flat code. The code_t type identifies where
101 the bit pattern counting takes place.
111 // Special data types.
112 typedef unsigned long long big_t; // type for code counting
113 #define PRIbig "llu" // printf format for big_t
114 typedef unsigned long long code_t; // type for bit pattern counting
115 struct tab { // type for been here check
116 size_t len; // length of bit vector in octets
117 char *vec; // allocated bit vector
120 /* The array for saving results, num[], is indexed with this triplet:
122 syms: number of symbols remaining to code
123 left: number of available bit patterns at length len
124 len: number of bits in the codes currently being assigned
126 Those indices are constrained thusly when saving results:
128 syms: 3..totsym (totsym == total symbols to code)
129 left: 2..syms - 1, but only the evens (so syms == 8 -> 2, 4, 6)
130 len: 1..max - 1 (max == maximum code length in bits)
132 syms == 2 is not saved since that immediately leads to a single code. left
133 must be even, since it represents the number of available bit patterns at
134 the current length, which is double the number at the previous length. left
135 ends at syms-1 since left == syms immediately results in a single code.
136 (left > sym is not allowed since that would result in an incomplete code.)
137 len is less than max, since the code completes immediately when len == max.
139 The offset into the array is calculated for the three indices with the first
140 one (syms) being outermost, and the last one (len) being innermost. We build
141 the array with length max-1 lists for the len index, with syms-3 of those
142 for each symbol. There are totsym-2 of those, with each one varying in
143 length as a function of sym. See the calculation of index in map() for the
144 index, and the calculation of size in main() for the size of the array.
146 For the deflate example of 286 symbols limited to 15-bit codes, the array
147 has 284,284 entries, taking up 2.17 MB for an 8-byte big_t. More than half
148 of the space allocated for saved results is actually used -- not all
149 possible triplets are reached in the generation of valid prefix codes.
152 /* The array for tracking visited states, done[], is itself indexed identically
153 to the num[] array as described above for the (syms, left, len) triplet.
154 Each element in the array is further indexed by the (mem, rem) doublet,
155 where mem is the amount of inflate table space used so far, and rem is the
156 remaining unused entries in the current inflate sub-table. Each indexed
157 element is simply one bit indicating whether the state has been visited or
158 not. Since the ranges for mem and rem are not known a priori, each bit
159 vector is of a variable size, and grows as needed to accommodate the visited
160 states. mem and rem are used to calculate a single index in a triangular
161 array. Since the range of mem is expected in the default case to be about
162 ten times larger than the range of rem, the array is skewed to reduce the
163 memory usage, with eight times the range for mem than for rem. See the
164 calculations for offset and bit in beenhere() for the details.
166 For the deflate example of 286 symbols limited to 15-bit codes, the bit
167 vectors grow to total approximately 21 MB, in addition to the 4.3 MB done[]
171 // Globals to avoid propagating constants or constant pointers recursively.
173 int max; // maximum allowed bit length for the codes
174 int root; // size of base code table in bits
175 int large; // largest code table so far
176 size_t size; // number of elements in num and done
177 int *code; // number of symbols assigned to each bit length
178 big_t *num; // saved results array for code counting
179 struct tab *done; // states already evaluated array
182 // Index function for num[] and done[].
183 local inline size_t map(int i, int j, int k) {
184 return k - 1 + ((size_t)((i - 1) >> 1) * ((i - 2) >> 1) + (j >> 1) - 1) *
188 // Free allocated space. Uses globals code, num, and done.
189 local void cleanup(void) {
192 if (g.done != NULL) {
193 for (n = 0; n < g.size; n++)
204 // Return the number of possible prefix codes using bit patterns of lengths len
205 // through max inclusive, coding syms symbols, with left bit patterns of length
206 // len unused -- return -1 if there is an overflow in the counting. Keep a
207 // record of previous results in num to prevent repeating the same calculation.
208 // Uses the globals max and num.
209 local big_t count(int syms, int len, int left) {
210 big_t sum; // number of possible codes from this juncture
211 big_t got; // value returned from count()
212 int least; // least number of syms to use at this juncture
213 int most; // most number of syms to use at this juncture
214 int use; // number of bit patterns to use in next call
215 size_t index; // index of this case in *num
217 // see if only one possible code
221 // note and verify the expected state
222 assert(syms > left && left > 0 && len < g.max);
224 // see if we've done this one already
225 index = map(syms, left, len);
228 return got; // we have -- return the saved result
230 // we need to use at least this many bit patterns so that the code won't be
231 // incomplete at the next length (more bit patterns than symbols)
232 least = (left << 1) - syms;
236 // we can use at most this many bit patterns, lest there not be enough
237 // available for the remaining symbols at the maximum length (if there were
238 // no limit to the code length, this would become: most = left - 1)
239 most = (((code_t)left << (g.max - len)) - syms) /
240 (((code_t)1 << (g.max - len)) - 1);
242 // count all possible codes from this juncture and add them up
244 for (use = least; use <= most; use++) {
245 got = count(syms - use, len + 1, (left - use) << 1);
247 if (got == (big_t)0 - 1 || sum < got) // overflow
251 // verify that all recursive calls are productive
254 // save the result and return it
259 // Return true if we've been here before, set to true if not. Set a bit in a
260 // bit vector to indicate visiting this state. Each (syms,len,left) state has a
261 // variable size bit vector indexed by (mem,rem). The bit vector is lengthened
262 // if needed to allow setting the (mem,rem) bit.
263 local int beenhere(int syms, int len, int left, int mem, int rem) {
264 size_t index; // index for this state's bit vector
265 size_t offset; // offset in this state's bit vector
266 int bit; // mask for this state's bit
267 size_t length; // length of the bit vector in bytes
268 char *vector; // new or enlarged bit vector
270 // point to vector for (syms,left,len), bit in vector for (mem,rem)
271 index = map(syms, left, len);
273 offset = (mem >> 3) + rem;
274 offset = ((offset * (offset + 1)) >> 1) + rem;
275 bit = 1 << (mem & 7);
277 // see if we've been here
278 length = g.done[index].len;
279 if (offset < length && (g.done[index].vec[offset] & bit) != 0)
280 return 1; // done this!
282 // we haven't been here before -- set the bit to show we have now
284 // see if we need to lengthen the vector in order to set the bit
285 if (length <= offset) {
286 // if we have one already, enlarge it, zero out the appended space
290 } while (length <= offset);
291 vector = realloc(g.done[index].vec, length);
293 memset(vector + g.done[index].len, 0,
294 length - g.done[index].len);
297 // otherwise we need to make a new vector and zero it out
299 length = 1 << (len - g.root);
300 while (length <= offset)
302 vector = calloc(length, sizeof(char));
305 // in either case, bail if we can't get the memory
306 if (vector == NULL) {
307 fputs("abort: unable to allocate enough memory\n", stderr);
312 // install the new vector
313 g.done[index].len = length;
314 g.done[index].vec = vector;
318 g.done[index].vec[offset] |= bit;
322 // Examine all possible codes from the given node (syms, len, left). Compute
323 // the amount of memory required to build inflate's decoding tables, where the
324 // number of code structures used so far is mem, and the number remaining in
325 // the current sub-table is rem. Uses the globals max, code, root, large, and
327 local void examine(int syms, int len, int left, int mem, int rem) {
328 int least; // least number of syms to use at this juncture
329 int most; // most number of syms to use at this juncture
330 int use; // number of bit patterns to use in next call
332 // see if we have a complete code
334 // set the last code entry
337 // complete computation of memory used by this code
340 rem = 1 << (len - g.root);
345 // if this is a new maximum, show the entries used and the sub-code
348 printf("max %d: ", mem);
349 for (use = g.root + 1; use <= g.max; use++)
351 printf("%d[%d] ", g.code[use], use);
356 // remove entries as we drop back down in the recursion
361 // prune the tree if we can
362 if (beenhere(syms, len, left, mem, rem))
365 // we need to use at least this many bit patterns so that the code won't be
366 // incomplete at the next length (more bit patterns than symbols)
367 least = (left << 1) - syms;
371 // we can use at most this many bit patterns, lest there not be enough
372 // available for the remaining symbols at the maximum length (if there were
373 // no limit to the code length, this would become: most = left - 1)
374 most = (((code_t)left << (g.max - len)) - syms) /
375 (((code_t)1 << (g.max - len)) - 1);
377 // occupy least table spaces, creating new sub-tables as needed
381 rem = 1 << (len - g.root);
386 // examine codes from here, updating table space as we go
387 for (use = least; use <= most; use++) {
389 examine(syms - use, len + 1, (left - use) << 1,
390 mem + (rem ? 1 << (len - g.root) : 0), rem << 1);
392 rem = 1 << (len - g.root);
398 // remove entries as we drop back down in the recursion
402 // Look at all sub-codes starting with root + 1 bits. Look at only the valid
403 // intermediate code states (syms, left, len). For each completed code,
404 // calculate the amount of memory required by inflate to build the decoding
405 // tables. Find the maximum amount of memory required and show the code that
406 // requires that maximum. Uses the globals max, root, and num.
407 local void enough(int syms) {
408 int n; // number of remaing symbols for this node
409 int left; // number of unused bit patterns at this length
410 size_t index; // index of this case in *num
413 for (n = 0; n <= g.max; n++)
416 // look at all (root + 1) bit and longer codes
417 g.large = 1 << g.root; // base table
418 if (g.root < g.max) // otherwise, there's only a base table
419 for (n = 3; n <= syms; n++)
420 for (left = 2; left < n; left += 2) {
421 // look at all reachable (root + 1) bit nodes, and the
422 // resulting codes (complete at root + 2 or more)
423 index = map(n, left, g.root + 1);
424 if (g.root + 1 < g.max && g.num[index]) // reachable node
425 examine(n, g.root + 1, left, 1 << g.root, 0);
427 // also look at root bit codes with completions at root + 1
428 // bits (not saved in num, since complete), just in case
429 if (g.num[index - 1] && n <= left << 1)
430 examine((n - left) << 1, g.root + 1, (n - left) << 1,
435 printf("done: maximum of %d table entries\n", g.large);
438 // Examine and show the total number of possible prefix codes for a given
439 // maximum number of symbols, initial root table size, and maximum code length
440 // in bits -- those are the command arguments in that order. The default values
441 // are 286, 9, and 15 respectively, for the deflate literal/length code. The
442 // possible codes are counted for each number of coded symbols from two to the
443 // maximum. The counts for each of those and the total number of codes are
444 // shown. The maximum number of inflate table entires is then calculated across
445 // all possible codes. Each new maximum number of table entries and the
446 // associated sub-code (starting at root + 1 == 10 bits) is shown.
448 // To count and examine prefix codes that are not length-limited, provide a
449 // maximum length equal to the number of symbols minus one.
451 // For the deflate literal/length code, use "enough". For the deflate distance
452 // code, use "enough 30 6".
453 int main(int argc, char **argv) {
454 int syms; // total number of symbols to code
455 int n; // number of symbols to code for this run
456 big_t got; // return value of count()
457 big_t sum; // accumulated number of codes over n
458 code_t word; // for counting bits in code_t
460 // set up globals for cleanup()
465 // get arguments -- default to the deflate literal/length code
470 syms = atoi(argv[1]);
472 g.root = atoi(argv[2]);
474 g.max = atoi(argv[3]);
477 if (argc > 4 || syms < 2 || g.root < 1 || g.max < 1) {
478 fputs("invalid arguments, need: [sym >= 2 [root >= 1 [max >= 1]]]\n",
483 // if not restricting the code length, the longest is syms - 1
484 if (g.max > syms - 1)
487 // determine the number of bits in a code_t
488 for (n = 0, word = 1; word; n++, word <<= 1)
491 // make sure that the calculation of most will not overflow
492 if (g.max > n || (code_t)(syms - 2) >= (((code_t)0 - 1) >> (g.max - 1))) {
493 fputs("abort: code length too long for internal types\n", stderr);
497 // reject impossible code requests
498 if ((code_t)(syms - 1) > ((code_t)1 << g.max) - 1) {
499 fprintf(stderr, "%d symbols cannot be coded in %d bits\n",
504 // allocate code vector
505 g.code = calloc(g.max + 1, sizeof(int));
506 if (g.code == NULL) {
507 fputs("abort: unable to allocate enough memory\n", stderr);
511 // determine size of saved results array, checking for overflows,
512 // allocate and clear the array (set all to zero with calloc())
513 if (syms == 2) // iff max == 1
514 g.num = NULL; // won't be saving any results
517 if (g.size > ((size_t)0 - 1) / (n = (syms - 1) >> 1) ||
518 (g.size *= n, g.size > ((size_t)0 - 1) / (n = g.max - 1)) ||
519 (g.size *= n, g.size > ((size_t)0 - 1) / sizeof(big_t)) ||
520 (g.num = calloc(g.size, sizeof(big_t))) == NULL) {
521 fputs("abort: unable to allocate enough memory\n", stderr);
527 // count possible codes for all numbers of symbols, add up counts
529 for (n = 2; n <= syms; n++) {
530 got = count(n, 1, 2);
532 if (got == (big_t)0 - 1 || sum < got) { // overflow
533 fputs("abort: can't count that high!\n", stderr);
537 printf("%"PRIbig" %d-codes\n", got, n);
539 printf("%"PRIbig" total codes for 2 to %d symbols", sum, syms);
540 if (g.max < syms - 1)
541 printf(" (%d-bit length limit)\n", g.max);
543 puts(" (no length limit)");
545 // allocate and clear done array for beenhere()
548 else if (g.size > ((size_t)0 - 1) / sizeof(struct tab) ||
549 (g.done = calloc(g.size, sizeof(struct tab))) == NULL) {
550 fputs("abort: unable to allocate enough memory\n", stderr);
555 // find and show maximum inflate table usage
556 if (g.root > g.max) // reduce root to max length
558 if ((code_t)syms < ((code_t)1 << (g.root + 1)))
561 puts("cannot handle minimum code lengths > root");