1 | /* $Id: bldprog-strtab-template.cpp.h 98103 2023-01-17 14:15:46Z vboxsync $ */
|
---|
2 | /** @file
|
---|
3 | * IPRT - Build Program - String Table Generator.
|
---|
4 | */
|
---|
5 |
|
---|
6 | /*
|
---|
7 | * Copyright (C) 2006-2023 Oracle and/or its affiliates.
|
---|
8 | *
|
---|
9 | * This file is part of VirtualBox base platform packages, as
|
---|
10 | * available from https://www.virtualbox.org.
|
---|
11 | *
|
---|
12 | * This program is free software; you can redistribute it and/or
|
---|
13 | * modify it under the terms of the GNU General Public License
|
---|
14 | * as published by the Free Software Foundation, in version 3 of the
|
---|
15 | * License.
|
---|
16 | *
|
---|
17 | * This program is distributed in the hope that it will be useful, but
|
---|
18 | * WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
---|
20 | * General Public License for more details.
|
---|
21 | *
|
---|
22 | * You should have received a copy of the GNU General Public License
|
---|
23 | * along with this program; if not, see <https://www.gnu.org/licenses>.
|
---|
24 | *
|
---|
25 | * The contents of this file may alternatively be used under the terms
|
---|
26 | * of the Common Development and Distribution License Version 1.0
|
---|
27 | * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
|
---|
28 | * in the VirtualBox distribution, in which case the provisions of the
|
---|
29 | * CDDL are applicable instead of those of the GPL.
|
---|
30 | *
|
---|
31 | * You may elect to license modified versions of this file under the
|
---|
32 | * terms and conditions of either the GPL or the CDDL or both.
|
---|
33 | *
|
---|
34 | * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
|
---|
35 | */
|
---|
36 |
|
---|
37 |
|
---|
38 | /*
|
---|
39 | * (Avoid include sanity checks as this is ring-3 only, C++ code.)
|
---|
40 | */
|
---|
41 | #if defined(__cplusplus) && defined(IN_RING3)
|
---|
42 |
|
---|
43 |
|
---|
44 | /*********************************************************************************************************************************
|
---|
45 | * Defined Constants And Macros *
|
---|
46 | *********************************************************************************************************************************/
|
---|
47 | /** @def BLDPROG_STRTAB_MAX_STRLEN
|
---|
48 | * The max length of strings in the table. */
|
---|
49 | #if !defined(BLDPROG_STRTAB_MAX_STRLEN) || defined(DOXYGEN_RUNNING)
|
---|
50 | # define BLDPROG_STRTAB_MAX_STRLEN 256
|
---|
51 | #endif
|
---|
52 |
|
---|
53 | /** @def BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
54 | * Enables very simple string compression.
|
---|
55 | */
|
---|
56 | #if defined(DOXYGEN_RUNNING)
|
---|
57 | # define BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
58 | #endif
|
---|
59 |
|
---|
60 | /** @def BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
61 | * Modifies the string compression to look for camel case words.
|
---|
62 | */
|
---|
63 | #if defined(DOXYGEN_RUNNING)
|
---|
64 | # define BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
65 | #endif
|
---|
66 |
|
---|
67 | /** @def BLDPROG_STRTAB_PURE_ASCII
|
---|
68 | * String compression assumes pure 7-bit ASCII and will fail on UTF-8 when this
|
---|
69 | * is defined. Otherwise, the compression code will require IPRT to link.
|
---|
70 | */
|
---|
71 | #if defined(DOXYGEN_RUNNING)
|
---|
72 | # define BLDPROG_STRTAB_PURE_ASCII
|
---|
73 | #endif
|
---|
74 |
|
---|
75 |
|
---|
76 |
|
---|
77 | /*********************************************************************************************************************************
|
---|
78 | * Header Files *
|
---|
79 | *********************************************************************************************************************************/
|
---|
80 | #include <iprt/stdarg.h>
|
---|
81 | #include <iprt/ctype.h>
|
---|
82 | #include <stdlib.h>
|
---|
83 | #include <stdio.h>
|
---|
84 | #include <string.h>
|
---|
85 |
|
---|
86 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
87 | # include <map>
|
---|
88 | # include <iprt/sanitized/string>
|
---|
89 |
|
---|
90 | # define BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
91 | typedef struct BLDPROGWORDFREQSTATS
|
---|
92 | {
|
---|
93 | uint32_t cWithoutSep; /**< Number of occurances without a separator. */
|
---|
94 | # ifdef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
95 | uint32_t cWithSep; /**< Number of occurance with a separator. */
|
---|
96 | char chSep; /**< The separator. First come basis. */
|
---|
97 | # endif
|
---|
98 | } BLDPROGWORDFREQSTATS;
|
---|
99 |
|
---|
100 | typedef std::map<std::string, BLDPROGWORDFREQSTATS> BLDPROGWORDFREQMAP;
|
---|
101 |
|
---|
102 | #endif
|
---|
103 |
|
---|
104 | #include "../../src/VBox/Runtime/include/internal/strhash.h" /** @todo make this one public */
|
---|
105 |
|
---|
106 |
|
---|
107 | /*********************************************************************************************************************************
|
---|
108 | * Structures and Typedefs *
|
---|
109 | *********************************************************************************************************************************/
|
---|
110 | /**
|
---|
111 | * Build table string.
|
---|
112 | */
|
---|
113 | typedef struct BLDPROGSTRING
|
---|
114 | {
|
---|
115 | /** The string.
|
---|
116 | * @note This may be modified or replaced (allocated from heap) when
|
---|
117 | * compressing the string table. */
|
---|
118 | char *pszString;
|
---|
119 | /** The string hash value. */
|
---|
120 | uint32_t uHash;
|
---|
121 | /** The strint table offset. */
|
---|
122 | uint32_t offStrTab;
|
---|
123 | /** The string length. */
|
---|
124 | size_t cchString;
|
---|
125 | /** Pointer to the next string reference (same string table entry). */
|
---|
126 | struct BLDPROGSTRING *pNextRef;
|
---|
127 | /** Pointer to the next string with the same hash value (collision). */
|
---|
128 | struct BLDPROGSTRING *pNextCollision;
|
---|
129 |
|
---|
130 | } BLDPROGSTRING;
|
---|
131 | /** Pointer to a string table string. */
|
---|
132 | typedef BLDPROGSTRING *PBLDPROGSTRING;
|
---|
133 |
|
---|
134 |
|
---|
135 | /** String table data. */
|
---|
136 | typedef struct BLDPROGSTRTAB
|
---|
137 | {
|
---|
138 | /** The size of g_papStrHash. */
|
---|
139 | size_t cStrHash;
|
---|
140 | /** String hash table. */
|
---|
141 | PBLDPROGSTRING *papStrHash;
|
---|
142 | /** Duplicate strings found by AddString. */
|
---|
143 | size_t cDuplicateStrings;
|
---|
144 | /** Total length of the unique strings (no terminators). */
|
---|
145 | size_t cchUniqueStrings;
|
---|
146 | /** Number of unique strings after AddString. */
|
---|
147 | size_t cUniqueStrings;
|
---|
148 | /** Number of collisions. */
|
---|
149 | size_t cCollisions;
|
---|
150 |
|
---|
151 | /** Number of entries in apSortedStrings. */
|
---|
152 | size_t cSortedStrings;
|
---|
153 | /** The sorted string table. */
|
---|
154 | PBLDPROGSTRING *papSortedStrings;
|
---|
155 |
|
---|
156 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
157 | /** The 256 words we've picked to be indexed by reference. */
|
---|
158 | BLDPROGSTRING aCompDict[256];
|
---|
159 | /** The frequency of the 256 dictionary entries. */
|
---|
160 | size_t auCompDictFreq[256];
|
---|
161 | /** Incoming strings pending compression. */
|
---|
162 | PBLDPROGSTRING *papPendingStrings;
|
---|
163 | /** Current number of entries in papStrPending. */
|
---|
164 | size_t cPendingStrings;
|
---|
165 | /** The allocated size of papPendingStrings. */
|
---|
166 | size_t cMaxPendingStrings;
|
---|
167 | /** Work frequency map.
|
---|
168 | * @todo rewrite in plain C. */
|
---|
169 | BLDPROGWORDFREQMAP Frequencies;
|
---|
170 | /** Map of characters used by input strings. */
|
---|
171 | uint64_t bmUsedChars[256/64];
|
---|
172 | #endif
|
---|
173 |
|
---|
174 | /** The string table. */
|
---|
175 | char *pachStrTab;
|
---|
176 | /** The actual string table size. */
|
---|
177 | size_t cchStrTab;
|
---|
178 | } BLDPROGSTRTAB;
|
---|
179 | typedef BLDPROGSTRTAB *PBLDPROGSTRTAB;
|
---|
180 |
|
---|
181 | #if RT_CLANG_PREREQ(4, 0) || RT_GNUC_PREREQ(4, 6)
|
---|
182 | # pragma GCC diagnostic push
|
---|
183 | # pragma GCC diagnostic ignored "-Wunused-function"
|
---|
184 | #endif
|
---|
185 |
|
---|
186 |
|
---|
187 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
188 |
|
---|
189 | /**
|
---|
190 | * Same as ASMBitTest.
|
---|
191 | *
|
---|
192 | * We cannot safely use ASMBitTest here because it must be inline, as this code
|
---|
193 | * is used to build RuntimeBldProg. */
|
---|
194 | DECLINLINE(bool) BldProgBitIsSet(uint64_t const *pbmBitmap, size_t iBit)
|
---|
195 | {
|
---|
196 | return RT_BOOL(pbmBitmap[iBit / 64] & RT_BIT_64(iBit % 64));
|
---|
197 | }
|
---|
198 |
|
---|
199 |
|
---|
200 | /**
|
---|
201 | * Same as ASMBitSet.
|
---|
202 | *
|
---|
203 | * We cannot safely use ASMBitSet here because it must be inline, as this code
|
---|
204 | * is used to build RuntimeBldProg.
|
---|
205 | */
|
---|
206 | DECLINLINE(void) BldProgBitSet(uint64_t *pbmBitmap, size_t iBit)
|
---|
207 | {
|
---|
208 | pbmBitmap[iBit / 64] |= RT_BIT_64(iBit % 64);
|
---|
209 | }
|
---|
210 |
|
---|
211 | #endif
|
---|
212 |
|
---|
213 |
|
---|
214 | /**
|
---|
215 | * Initializes the strint table compiler.
|
---|
216 | *
|
---|
217 | * @returns success indicator (out of memory if false).
|
---|
218 | * @param pThis The strint table compiler instance.
|
---|
219 | * @param cMaxStrings The max number of strings we'll be adding.
|
---|
220 | */
|
---|
221 | static bool BldProgStrTab_Init(PBLDPROGSTRTAB pThis, size_t cMaxStrings)
|
---|
222 | {
|
---|
223 | pThis->cStrHash = 0;
|
---|
224 | pThis->papStrHash = NULL;
|
---|
225 | pThis->cDuplicateStrings = 0;
|
---|
226 | pThis->cchUniqueStrings = 0;
|
---|
227 | pThis->cUniqueStrings = 0;
|
---|
228 | pThis->cCollisions = 0;
|
---|
229 | pThis->cSortedStrings = 0;
|
---|
230 | pThis->papSortedStrings = NULL;
|
---|
231 | pThis->pachStrTab = NULL;
|
---|
232 | pThis->cchStrTab = 0;
|
---|
233 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
234 | memset(pThis->aCompDict, 0, sizeof(pThis->aCompDict));
|
---|
235 | pThis->papPendingStrings = NULL;
|
---|
236 | pThis->cPendingStrings = 0;
|
---|
237 | pThis->cMaxPendingStrings = cMaxStrings;
|
---|
238 | memset(pThis->bmUsedChars, 0, sizeof(pThis->bmUsedChars));
|
---|
239 | BldProgBitSet(pThis->bmUsedChars, 0); /* Some parts of the code still thinks zero is a terminator, so don't use it for now. */
|
---|
240 | # ifndef BLDPROG_STRTAB_PURE_ASCII
|
---|
241 | BldProgBitSet(pThis->bmUsedChars, 0xff); /* Reserve escape byte for codepoints above 127. */
|
---|
242 | # endif
|
---|
243 | #endif
|
---|
244 |
|
---|
245 | /*
|
---|
246 | * Allocate a hash table double the size of all strings (to avoid too
|
---|
247 | * many collisions). Add all strings to it, finding duplicates in the
|
---|
248 | * process.
|
---|
249 | */
|
---|
250 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
251 | cMaxStrings += RT_ELEMENTS(pThis->aCompDict);
|
---|
252 | #endif
|
---|
253 | cMaxStrings *= 2;
|
---|
254 | pThis->papStrHash = (PBLDPROGSTRING *)calloc(sizeof(pThis->papStrHash[0]), cMaxStrings);
|
---|
255 | if (pThis->papStrHash)
|
---|
256 | {
|
---|
257 | pThis->cStrHash = cMaxStrings;
|
---|
258 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
259 | pThis->papPendingStrings = (PBLDPROGSTRING *)calloc(sizeof(pThis->papPendingStrings[0]), pThis->cMaxPendingStrings);
|
---|
260 | if (pThis->papPendingStrings)
|
---|
261 | #endif
|
---|
262 | return true;
|
---|
263 |
|
---|
264 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
265 | free(pThis->papStrHash);
|
---|
266 | pThis->papStrHash = NULL;
|
---|
267 | #endif
|
---|
268 | }
|
---|
269 | return false;
|
---|
270 | }
|
---|
271 |
|
---|
272 |
|
---|
273 | #if 0 /* unused */
|
---|
274 | static void BldProgStrTab_Delete(PBLDPROGSTRTAB pThis)
|
---|
275 | {
|
---|
276 | free(pThis->papStrHash);
|
---|
277 | free(pThis->papSortedStrings);
|
---|
278 | free(pThis->pachStrTab);
|
---|
279 | # ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
280 | free(pThis->papPendingStrings);
|
---|
281 | # endif
|
---|
282 | memset(pThis, 0, sizeof(*pThis));
|
---|
283 | }
|
---|
284 | #endif
|
---|
285 |
|
---|
286 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
287 |
|
---|
288 | DECLINLINE(size_t) bldProgStrTab_compressorFindNextWord(const char *pszSrc, char ch, const char **ppszSrc)
|
---|
289 | {
|
---|
290 | /*
|
---|
291 | * Skip leading word separators.
|
---|
292 | */
|
---|
293 | # ifdef BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
294 | while ( ch == ' '
|
---|
295 | || ch == '-'
|
---|
296 | || ch == '+'
|
---|
297 | || ch == '_')
|
---|
298 | # else
|
---|
299 | while (ch == ' ')
|
---|
300 | # endif
|
---|
301 | ch = *++pszSrc;
|
---|
302 | if (ch)
|
---|
303 | {
|
---|
304 | /*
|
---|
305 | * Find end of word.
|
---|
306 | */
|
---|
307 | size_t cchWord = 1;
|
---|
308 | # ifdef BLDPROG_STRTAB_WITH_CAMEL_WORDS
|
---|
309 | char chPrev = ch;
|
---|
310 | while ( (ch = pszSrc[cchWord]) != ' '
|
---|
311 | && ch != '\0'
|
---|
312 | && ch != '-'
|
---|
313 | && ch != '+'
|
---|
314 | && ch != '_'
|
---|
315 | && ( ch == chPrev
|
---|
316 | || !RT_C_IS_UPPER(ch)
|
---|
317 | || RT_C_IS_UPPER(chPrev)) )
|
---|
318 | {
|
---|
319 | chPrev = ch;
|
---|
320 | cchWord++;
|
---|
321 | }
|
---|
322 | # else
|
---|
323 | while ((ch = pszSrc[cchWord]) != ' ' && ch != '\0')
|
---|
324 | cchWord++;
|
---|
325 | # endif
|
---|
326 | *ppszSrc = pszSrc;
|
---|
327 | return cchWord;
|
---|
328 | }
|
---|
329 |
|
---|
330 | *ppszSrc = pszSrc;
|
---|
331 | return 0;
|
---|
332 | }
|
---|
333 |
|
---|
334 |
|
---|
335 | /**
|
---|
336 | * Analyzes a string.
|
---|
337 | *
|
---|
338 | * @param pThis The strint table compiler instance.
|
---|
339 | * @param pStr The string to analyze.
|
---|
340 | */
|
---|
341 | static void bldProgStrTab_compressorAnalyzeString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
342 | {
|
---|
343 | /*
|
---|
344 | * Mark all the string characters as used.
|
---|
345 | */
|
---|
346 | const char *psz = pStr->pszString;
|
---|
347 | char ch;
|
---|
348 | while ((ch = *psz++) != '\0')
|
---|
349 | BldProgBitSet(pThis->bmUsedChars, (uint8_t)ch);
|
---|
350 |
|
---|
351 | /*
|
---|
352 | * For now we just consider words.
|
---|
353 | */
|
---|
354 | psz = pStr->pszString;
|
---|
355 | while ((ch = *psz) != '\0')
|
---|
356 | {
|
---|
357 | size_t cchWord = bldProgStrTab_compressorFindNextWord(psz, ch, &psz);
|
---|
358 | if (cchWord > 1)
|
---|
359 | {
|
---|
360 | std::string strWord(psz, cchWord);
|
---|
361 | BLDPROGWORDFREQMAP::iterator it = pThis->Frequencies.find(strWord);
|
---|
362 | if (it != pThis->Frequencies.end())
|
---|
363 | {
|
---|
364 | # ifdef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
365 | char const chSep = psz[cchWord];
|
---|
366 | if (chSep != '\0' && (it->second.chSep == chSep || it->second.chSep == '\0'))
|
---|
367 | {
|
---|
368 | it->second.chSep = chSep;
|
---|
369 | it->second.cWithSep++;
|
---|
370 | }
|
---|
371 | else
|
---|
372 | # endif
|
---|
373 | it->second.cWithoutSep++;
|
---|
374 | }
|
---|
375 | else
|
---|
376 | {
|
---|
377 | # ifdef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
378 | char const chSep = psz[cchWord];
|
---|
379 | if (chSep != '\0')
|
---|
380 | {
|
---|
381 | BLDPROGWORDFREQSTATS const NewWord = { 0, 0, chSep };
|
---|
382 | pThis->Frequencies[strWord] = NewWord;
|
---|
383 | }
|
---|
384 | else
|
---|
385 | {
|
---|
386 | static BLDPROGWORDFREQSTATS const s_NewWord = { 0, 0, 0 };
|
---|
387 | pThis->Frequencies[strWord] = s_NewWord;
|
---|
388 | }
|
---|
389 | # else
|
---|
390 | pThis->Frequencies[strWord].cWithoutSep = 0;
|
---|
391 | # endif
|
---|
392 | }
|
---|
393 |
|
---|
394 | # if 0 /** @todo need better accounting for overlapping alternatives before this can be enabled. */
|
---|
395 | /* Two words - immediate yields calc may lie when this enabled and we may pick the wrong words. */
|
---|
396 | if (ch == ' ')
|
---|
397 | {
|
---|
398 | ch = psz[++cchWord];
|
---|
399 | if (ch != ' ' && ch != '\0')
|
---|
400 | {
|
---|
401 | size_t const cchSaved = cchWord;
|
---|
402 |
|
---|
403 | do
|
---|
404 | cchWord++;
|
---|
405 | while ((ch = psz[cchWord]) != ' ' && ch != '\0');
|
---|
406 |
|
---|
407 | strWord = std::string(psz, cchWord);
|
---|
408 | BLDPROGWORDFREQMAP::iterator it = pThis->Frequencies.find(strWord);
|
---|
409 | if (it != pThis->Frequencies.end())
|
---|
410 | it->second += cchWord - 1;
|
---|
411 | else
|
---|
412 | pThis->Frequencies[strWord] = 0;
|
---|
413 |
|
---|
414 | cchWord = cchSaved;
|
---|
415 | }
|
---|
416 | }
|
---|
417 | # endif
|
---|
418 | }
|
---|
419 | else if (!cchWord)
|
---|
420 | break;
|
---|
421 |
|
---|
422 | /* Advance. */
|
---|
423 | psz += cchWord;
|
---|
424 | }
|
---|
425 | pStr->cchString = psz - pStr->pszString;
|
---|
426 | if (pStr->cchString > BLDPROG_STRTAB_MAX_STRLEN)
|
---|
427 | {
|
---|
428 | fprintf(stderr, "error: String to long (%u)\n", (unsigned)pStr->cchString);
|
---|
429 | abort();
|
---|
430 | }
|
---|
431 | }
|
---|
432 |
|
---|
433 | #endif /* BLDPROG_STRTAB_WITH_COMPRESSION */
|
---|
434 |
|
---|
435 | /**
|
---|
436 | * Adds a string to the hash table.
|
---|
437 | * @param pThis The strint table compiler instance.
|
---|
438 | * @param pStr The string.
|
---|
439 | */
|
---|
440 | static void bldProgStrTab_AddStringToHashTab(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
441 | {
|
---|
442 | pStr->pNextRef = NULL;
|
---|
443 | pStr->pNextCollision = NULL;
|
---|
444 | pStr->offStrTab = 0;
|
---|
445 | pStr->uHash = sdbm(pStr->pszString, &pStr->cchString);
|
---|
446 | if (pStr->cchString > BLDPROG_STRTAB_MAX_STRLEN)
|
---|
447 | {
|
---|
448 | fprintf(stderr, "error: String to long (%u)\n", (unsigned)pStr->cchString);
|
---|
449 | exit(RTEXITCODE_FAILURE);
|
---|
450 | }
|
---|
451 |
|
---|
452 | size_t idxHash = pStr->uHash % pThis->cStrHash;
|
---|
453 | PBLDPROGSTRING pCur = pThis->papStrHash[idxHash];
|
---|
454 | if (!pCur)
|
---|
455 | pThis->papStrHash[idxHash] = pStr;
|
---|
456 | else
|
---|
457 | {
|
---|
458 | /* Look for matching string. */
|
---|
459 | do
|
---|
460 | {
|
---|
461 | if ( pCur->uHash == pStr->uHash
|
---|
462 | && pCur->cchString == pStr->cchString
|
---|
463 | && memcmp(pCur->pszString, pStr->pszString, pStr->cchString) == 0)
|
---|
464 | {
|
---|
465 | pStr->pNextRef = pCur->pNextRef;
|
---|
466 | pCur->pNextRef = pStr;
|
---|
467 | pThis->cDuplicateStrings++;
|
---|
468 | return;
|
---|
469 | }
|
---|
470 | pCur = pCur->pNextCollision;
|
---|
471 | } while (pCur != NULL);
|
---|
472 |
|
---|
473 | /* No matching string, insert. */
|
---|
474 | pThis->cCollisions++;
|
---|
475 | pStr->pNextCollision = pThis->papStrHash[idxHash];
|
---|
476 | pThis->papStrHash[idxHash] = pStr;
|
---|
477 | }
|
---|
478 |
|
---|
479 | pThis->cUniqueStrings++;
|
---|
480 | pThis->cchUniqueStrings += pStr->cchString;
|
---|
481 | }
|
---|
482 |
|
---|
483 |
|
---|
484 | /**
|
---|
485 | * Adds a string to the string table.
|
---|
486 | *
|
---|
487 | * @param pThis The strint table compiler instance.
|
---|
488 | * @param pStr The string.
|
---|
489 | */
|
---|
490 | static void BldProgStrTab_AddString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
491 | {
|
---|
492 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
493 | bldProgStrTab_compressorAnalyzeString(pThis, pStr);
|
---|
494 | if (pThis->cPendingStrings < pThis->cMaxPendingStrings)
|
---|
495 | pThis->papPendingStrings[pThis->cPendingStrings++] = pStr;
|
---|
496 | else
|
---|
497 | abort();
|
---|
498 | #else
|
---|
499 | bldProgStrTab_AddStringToHashTab(pThis, pStr);
|
---|
500 | #endif
|
---|
501 | }
|
---|
502 |
|
---|
503 |
|
---|
504 | /**
|
---|
505 | * Adds a string to the string table.
|
---|
506 | *
|
---|
507 | * @param pThis The strint table compiler instance.
|
---|
508 | * @param pStr The string entry (uninitialized).
|
---|
509 | * @param psz The string, will be duplicated if compression is enabled.
|
---|
510 | */
|
---|
511 | DECLINLINE(void) BldProgStrTab_AddStringDup(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr, const char *psz)
|
---|
512 | {
|
---|
513 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
514 | pStr->pszString = strdup(psz);
|
---|
515 | if (!pStr->pszString)
|
---|
516 | abort();
|
---|
517 | #else
|
---|
518 | pStr->pszString = (char *)psz;
|
---|
519 | #endif
|
---|
520 | BldProgStrTab_AddString(pThis, pStr);
|
---|
521 | }
|
---|
522 |
|
---|
523 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
524 |
|
---|
525 | /**
|
---|
526 | * Copies @a cchSrc chars from @a pchSrc to @a pszDst, escaping special
|
---|
527 | * sequences.
|
---|
528 | *
|
---|
529 | * @returns New @a pszDst position, NULL if invalid source encoding.
|
---|
530 | * @param pszDst The destination buffer.
|
---|
531 | * @param pszSrc The source buffer.
|
---|
532 | * @param cchSrc How much to copy.
|
---|
533 | */
|
---|
534 | static char *bldProgStrTab_compressorCopyAndEscape(char *pszDst, const char *pszSrc, size_t cchSrc)
|
---|
535 | {
|
---|
536 | while (cchSrc-- > 0)
|
---|
537 | {
|
---|
538 | char ch = *pszSrc;
|
---|
539 | if (!((unsigned char)ch & 0x80))
|
---|
540 | {
|
---|
541 | *pszDst++ = ch;
|
---|
542 | pszSrc++;
|
---|
543 | }
|
---|
544 | else
|
---|
545 | {
|
---|
546 | # ifdef BLDPROG_STRTAB_PURE_ASCII
|
---|
547 | fprintf(stderr, "error: unexpected char value %#x\n", ch);
|
---|
548 | return NULL;
|
---|
549 | # else
|
---|
550 | RTUNICP uc;
|
---|
551 | int rc = RTStrGetCpEx(&pszSrc, &uc);
|
---|
552 | if (RT_SUCCESS(rc))
|
---|
553 | {
|
---|
554 | *pszDst++ = (unsigned char)0xff; /* escape single code point. */
|
---|
555 | pszDst = RTStrPutCp(pszDst, uc);
|
---|
556 | }
|
---|
557 | else
|
---|
558 | {
|
---|
559 | fprintf(stderr, "Error: RTStrGetCpEx failed with rc=%d\n", rc);
|
---|
560 | return NULL;
|
---|
561 | }
|
---|
562 | # endif
|
---|
563 | }
|
---|
564 | }
|
---|
565 | return pszDst;
|
---|
566 | }
|
---|
567 |
|
---|
568 |
|
---|
569 | /**
|
---|
570 | * Replaces the dictionary words and escapes non-ascii chars in a string.
|
---|
571 | *
|
---|
572 | * @param pThis The strint table compiler instance.
|
---|
573 | * @param pString The string to fixup.
|
---|
574 | */
|
---|
575 | static bool bldProgStrTab_compressorFixupString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
576 | {
|
---|
577 | char szNew[BLDPROG_STRTAB_MAX_STRLEN * 2];
|
---|
578 | char *pszDst = szNew;
|
---|
579 | const char *pszSrc = pStr->pszString;
|
---|
580 | const char *pszSrcEnd = pszSrc + pStr->cchString;
|
---|
581 |
|
---|
582 | char ch;
|
---|
583 | while ((ch = *pszSrc) != '\0')
|
---|
584 | {
|
---|
585 | const char * const pszSrcUncompressed = pszSrc;
|
---|
586 | size_t cchWord = bldProgStrTab_compressorFindNextWord(pszSrc, ch, &pszSrc);
|
---|
587 | size_t cchSrcUncompressed = pszSrc - pszSrcUncompressed;
|
---|
588 | if (cchSrcUncompressed > 0)
|
---|
589 | {
|
---|
590 | pszDst = bldProgStrTab_compressorCopyAndEscape(pszDst, pszSrcUncompressed, cchSrcUncompressed);
|
---|
591 | if (!pszDst)
|
---|
592 | return false;
|
---|
593 | }
|
---|
594 | if (!cchWord)
|
---|
595 | break;
|
---|
596 |
|
---|
597 | /* Check for g_aWord matches. */
|
---|
598 | if (cchWord > 1)
|
---|
599 | {
|
---|
600 | size_t cchMax = pszSrcEnd - pszSrc;
|
---|
601 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
602 | {
|
---|
603 | size_t cchLen = pThis->aCompDict[i].cchString;
|
---|
604 | if ( cchLen >= cchWord
|
---|
605 | && cchLen <= cchMax
|
---|
606 | && memcmp(pThis->aCompDict[i].pszString, pszSrc, cchLen) == 0)
|
---|
607 | {
|
---|
608 | *pszDst++ = (unsigned char)i;
|
---|
609 | pszSrc += cchLen;
|
---|
610 | cchWord = 0;
|
---|
611 | break;
|
---|
612 | }
|
---|
613 | }
|
---|
614 | }
|
---|
615 |
|
---|
616 | if (cchWord > 0)
|
---|
617 | {
|
---|
618 | /* Copy the current word. */
|
---|
619 | pszDst = bldProgStrTab_compressorCopyAndEscape(pszDst, pszSrc, cchWord);
|
---|
620 | if (!pszDst)
|
---|
621 | return false;
|
---|
622 | pszSrc += cchWord;
|
---|
623 | }
|
---|
624 | }
|
---|
625 |
|
---|
626 | /* Just terminate it now. */
|
---|
627 | *pszDst = '\0';
|
---|
628 |
|
---|
629 | /*
|
---|
630 | * Update the string.
|
---|
631 | */
|
---|
632 | size_t cchNew = pszDst - &szNew[0];
|
---|
633 | if (cchNew > pStr->cchString)
|
---|
634 | {
|
---|
635 | pStr->pszString = (char *)malloc(cchNew + 1);
|
---|
636 | if (!pStr->pszString)
|
---|
637 | {
|
---|
638 | fprintf(stderr, "Out of memory!\n");
|
---|
639 | return false;
|
---|
640 | }
|
---|
641 | }
|
---|
642 | memcpy(pStr->pszString, szNew, cchNew + 1);
|
---|
643 | pStr->cchString = cchNew;
|
---|
644 |
|
---|
645 | return true;
|
---|
646 | }
|
---|
647 |
|
---|
648 |
|
---|
649 | /**
|
---|
650 | * Entry in SortedDictionary.
|
---|
651 | *
|
---|
652 | * Uses variable length string member, so not class and allocated via malloc.
|
---|
653 | */
|
---|
654 | struct SortedDictionaryEntry
|
---|
655 | {
|
---|
656 | size_t m_cchGain;
|
---|
657 | size_t m_cchString;
|
---|
658 | RT_FLEXIBLE_ARRAY_EXTENSION
|
---|
659 | char m_szString[RT_FLEXIBLE_ARRAY];
|
---|
660 |
|
---|
661 | /** Allocates and initializes a new entry. */
|
---|
662 | static SortedDictionaryEntry *allocate(const char *a_pch, size_t a_cch, size_t a_cchGain, char a_chSep)
|
---|
663 | {
|
---|
664 | size_t cbString = a_cch + !!a_chSep + 1;
|
---|
665 | SortedDictionaryEntry *pNew = (SortedDictionaryEntry *)malloc(RT_UOFFSETOF(SortedDictionaryEntry, m_szString) + cbString);
|
---|
666 | if (pNew)
|
---|
667 | {
|
---|
668 | pNew->m_cchGain = a_cchGain;
|
---|
669 | memcpy(pNew->m_szString, a_pch, a_cch);
|
---|
670 | if (a_chSep)
|
---|
671 | pNew->m_szString[a_cch++] = a_chSep;
|
---|
672 | pNew->m_szString[a_cch] = '\0';
|
---|
673 | pNew->m_cchString = a_cch;
|
---|
674 | }
|
---|
675 | return pNew;
|
---|
676 | }
|
---|
677 |
|
---|
678 |
|
---|
679 | /** Compares this dictionary entry with an incoming one.
|
---|
680 | * @retval -1 if this entry is of less worth than the new one.
|
---|
681 | * @retval 0 if this entry is of equal worth to the new one.
|
---|
682 | * @retval +1 if this entry is of more worth than the new one.
|
---|
683 | */
|
---|
684 | int compare(size_t a_cchGain, size_t a_cchString)
|
---|
685 | {
|
---|
686 | /* Higher gain is preferred of course: */
|
---|
687 | if (m_cchGain < a_cchGain)
|
---|
688 | return -1;
|
---|
689 | if (m_cchGain > a_cchGain)
|
---|
690 | return 1;
|
---|
691 |
|
---|
692 | /* Gain is the same. Prefer the shorter string, as it will result in a shorter string table: */
|
---|
693 | if (m_cchString > a_cchString)
|
---|
694 | return -1;
|
---|
695 | if (m_cchString < a_cchString)
|
---|
696 | return 1;
|
---|
697 | return 0;
|
---|
698 | }
|
---|
699 | };
|
---|
700 |
|
---|
701 |
|
---|
702 | /**
|
---|
703 | * Insertion sort dictionary that keeps the 256 best words.
|
---|
704 | *
|
---|
705 | * Used by bldProgStrTab_compressorDoStringCompression to pick the dictionary
|
---|
706 | * words.
|
---|
707 | */
|
---|
708 | class SortedDictionary
|
---|
709 | {
|
---|
710 | public:
|
---|
711 | size_t m_cEntries;
|
---|
712 | SortedDictionaryEntry *m_apEntries[256];
|
---|
713 |
|
---|
714 | SortedDictionary()
|
---|
715 | : m_cEntries(0)
|
---|
716 | {
|
---|
717 | for (size_t i = 0; i < RT_ELEMENTS(m_apEntries); i++)
|
---|
718 | m_apEntries[i] = NULL;
|
---|
719 | }
|
---|
720 |
|
---|
721 | ~SortedDictionary()
|
---|
722 | {
|
---|
723 | while (m_cEntries > 0)
|
---|
724 | {
|
---|
725 | free(m_apEntries[--m_cEntries]);
|
---|
726 | m_apEntries[m_cEntries] = NULL;
|
---|
727 | }
|
---|
728 | }
|
---|
729 |
|
---|
730 |
|
---|
731 | /**
|
---|
732 | * Inserts a new entry, if it's worth it.
|
---|
733 | * @returns true on succes, false if out of memory.
|
---|
734 | */
|
---|
735 | bool insert(const char *a_pchString, size_t a_cchStringBase, size_t a_cchGain, char a_chSep = 0)
|
---|
736 | {
|
---|
737 | size_t const cchString = a_cchStringBase + (a_chSep + 1);
|
---|
738 |
|
---|
739 | /*
|
---|
740 | * Drop the insert if the symbol table is full and the insert is less worth the last entry:
|
---|
741 | */
|
---|
742 | if ( m_cEntries >= RT_ELEMENTS(m_apEntries)
|
---|
743 | && m_apEntries[RT_ELEMENTS(m_apEntries) - 1]->compare(a_cchGain, cchString) >= 0)
|
---|
744 | return true;
|
---|
745 |
|
---|
746 | /*
|
---|
747 | * Create a new entry to insert.
|
---|
748 | */
|
---|
749 | SortedDictionaryEntry *pNewEntry = SortedDictionaryEntry::allocate(a_pchString, a_cchStringBase, a_cchGain, a_chSep);
|
---|
750 | if (!pNewEntry)
|
---|
751 | return false;
|
---|
752 |
|
---|
753 | /*
|
---|
754 | * Find the insert point.
|
---|
755 | */
|
---|
756 | if (m_cEntries == 0)
|
---|
757 | {
|
---|
758 | m_apEntries[0] = pNewEntry;
|
---|
759 | m_cEntries = 1;
|
---|
760 | }
|
---|
761 | else
|
---|
762 | {
|
---|
763 | /* If table is full, drop the last entry before we start (already made
|
---|
764 | sure the incoming entry is preferable to the one were dropping): */
|
---|
765 | if (m_cEntries >= RT_ELEMENTS(m_apEntries))
|
---|
766 | {
|
---|
767 | free(m_apEntries[RT_ELEMENTS(m_apEntries) - 1]);
|
---|
768 | m_apEntries[RT_ELEMENTS(m_apEntries) - 1] = NULL;
|
---|
769 | m_cEntries = RT_ELEMENTS(m_apEntries) - 1;
|
---|
770 | }
|
---|
771 |
|
---|
772 | /* Find where to insert the new entry: */
|
---|
773 | /** @todo use binary search. */
|
---|
774 | size_t i = m_cEntries;
|
---|
775 | while (i > 0 && m_apEntries[i - 1]->compare(a_cchGain, cchString) < 0)
|
---|
776 | i--;
|
---|
777 |
|
---|
778 | /* Shift entries to make room and insert the new entry. */
|
---|
779 | if (i < m_cEntries)
|
---|
780 | memmove(&m_apEntries[i + 1], &m_apEntries[i], (m_cEntries - i) * sizeof(m_apEntries[0]));
|
---|
781 | m_apEntries[i] = pNewEntry;
|
---|
782 | m_cEntries++;
|
---|
783 | }
|
---|
784 | return true;
|
---|
785 | }
|
---|
786 | };
|
---|
787 |
|
---|
788 |
|
---|
789 | /**
|
---|
790 | * Compresses the vendor and product strings.
|
---|
791 | *
|
---|
792 | * This is very very simple (a lot less work than the string table for
|
---|
793 | * instance).
|
---|
794 | */
|
---|
795 | static bool bldProgStrTab_compressorDoStringCompression(PBLDPROGSTRTAB pThis, bool fVerbose)
|
---|
796 | {
|
---|
797 | /*
|
---|
798 | * Sort the frequency analyzis result and pick the top entries for any
|
---|
799 | * available dictionary slots.
|
---|
800 | */
|
---|
801 | SortedDictionary SortedDict;
|
---|
802 | for (BLDPROGWORDFREQMAP::iterator it = pThis->Frequencies.begin(); it != pThis->Frequencies.end(); ++it)
|
---|
803 | {
|
---|
804 | bool fInsert;
|
---|
805 | size_t const cchString = it->first.length();
|
---|
806 | # ifndef BLDPROG_STRTAB_WITH_WORD_SEP_ALTERNATIVE
|
---|
807 | size_t const cchGainWithout = it->second.cWithoutSep * cchString;
|
---|
808 | # else
|
---|
809 | size_t const cchGainWithout = (it->second.cWithoutSep + it->second.cWithSep) * cchString;
|
---|
810 | size_t const cchGainWith = it->second.cWithSep * (cchString + 1);
|
---|
811 | if (cchGainWith > cchGainWithout)
|
---|
812 | fInsert = SortedDict.insert(it->first.c_str(), cchString, cchGainWith, it->second.chSep);
|
---|
813 | else
|
---|
814 | # endif
|
---|
815 | fInsert = SortedDict.insert(it->first.c_str(), cchString, cchGainWithout);
|
---|
816 | if (!fInsert)
|
---|
817 | return false;
|
---|
818 | }
|
---|
819 |
|
---|
820 | size_t cb = 0;
|
---|
821 | size_t cWords = 0;
|
---|
822 | size_t iDict = 0;
|
---|
823 | for (size_t i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
824 | {
|
---|
825 | char szTmp[2] = { (char)i, '\0' };
|
---|
826 | const char *psz = szTmp;
|
---|
827 | if ( BldProgBitIsSet(pThis->bmUsedChars, i)
|
---|
828 | || iDict >= SortedDict.m_cEntries)
|
---|
829 | {
|
---|
830 | /* character entry */
|
---|
831 | pThis->auCompDictFreq[i] = 0;
|
---|
832 | pThis->aCompDict[i].cchString = 1;
|
---|
833 | }
|
---|
834 | else
|
---|
835 | {
|
---|
836 | /* word entry */
|
---|
837 | cb += SortedDict.m_apEntries[iDict]->m_cchGain;
|
---|
838 | pThis->auCompDictFreq[i] = SortedDict.m_apEntries[iDict]->m_cchGain;
|
---|
839 | pThis->aCompDict[i].cchString = SortedDict.m_apEntries[iDict]->m_cchString;
|
---|
840 | psz = SortedDict.m_apEntries[iDict]->m_szString;
|
---|
841 | cWords++;
|
---|
842 | iDict++;
|
---|
843 | }
|
---|
844 | pThis->aCompDict[i].pszString = (char *)malloc(pThis->aCompDict[i].cchString + 1);
|
---|
845 | if (pThis->aCompDict[i].pszString)
|
---|
846 | memcpy(pThis->aCompDict[i].pszString, psz, pThis->aCompDict[i].cchString + 1);
|
---|
847 | else
|
---|
848 | return false;
|
---|
849 | }
|
---|
850 |
|
---|
851 | if (fVerbose)
|
---|
852 | printf("debug: Estimated string compression saving: %u bytes\n"
|
---|
853 | "debug: %u words, %u characters\n"
|
---|
854 | , (unsigned)cb, (unsigned)cWords, (unsigned)(RT_ELEMENTS(pThis->aCompDict) - cWords));
|
---|
855 |
|
---|
856 | /*
|
---|
857 | * Rework the strings.
|
---|
858 | */
|
---|
859 | size_t cchOld = 0;
|
---|
860 | size_t cchOldMax = 0;
|
---|
861 | size_t cchOldMin = BLDPROG_STRTAB_MAX_STRLEN;
|
---|
862 | size_t cchNew = 0;
|
---|
863 | size_t cchNewMax = 0;
|
---|
864 | size_t cchNewMin = BLDPROG_STRTAB_MAX_STRLEN;
|
---|
865 | size_t i = pThis->cPendingStrings;
|
---|
866 | while (i-- > 0)
|
---|
867 | {
|
---|
868 | PBLDPROGSTRING pCurStr = pThis->papPendingStrings[i];
|
---|
869 | cchOld += pCurStr->cchString;
|
---|
870 | if (pCurStr->cchString > cchOldMax)
|
---|
871 | cchOldMax = pCurStr->cchString;
|
---|
872 | if (pCurStr->cchString < cchOldMin)
|
---|
873 | cchOldMin = pCurStr->cchString;
|
---|
874 |
|
---|
875 | if (!bldProgStrTab_compressorFixupString(pThis, pCurStr))
|
---|
876 | return false;
|
---|
877 |
|
---|
878 | cchNew += pCurStr->cchString;
|
---|
879 | if (pCurStr->cchString > cchNewMax)
|
---|
880 | cchNewMax = pCurStr->cchString;
|
---|
881 | if (pCurStr->cchString < cchNewMin)
|
---|
882 | cchNewMin = pCurStr->cchString;
|
---|
883 |
|
---|
884 | bldProgStrTab_AddStringToHashTab(pThis, pCurStr);
|
---|
885 | }
|
---|
886 |
|
---|
887 | /*
|
---|
888 | * Do debug stats.
|
---|
889 | */
|
---|
890 | if (fVerbose)
|
---|
891 | {
|
---|
892 | for (i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
893 | cchNew += pThis->aCompDict[i].cchString + 1;
|
---|
894 |
|
---|
895 | printf("debug: Strings: original: %u bytes; compressed: %u bytes;", (unsigned)cchOld, (unsigned)cchNew);
|
---|
896 | if (cchNew < cchOld)
|
---|
897 | printf(" saving %u bytes (%u%%)\n", (unsigned)(cchOld - cchNew), (unsigned)((cchOld - cchNew) * 100 / cchOld));
|
---|
898 | else
|
---|
899 | printf(" wasting %u bytes!\n", (unsigned)(cchOld - cchNew));
|
---|
900 | printf("debug: Original string lengths: average %u; min %u; max %u\n",
|
---|
901 | (unsigned)(cchOld / pThis->cPendingStrings), (unsigned)cchOldMin, (unsigned)cchOldMax);
|
---|
902 | printf("debug: Compressed string lengths: average %u; min %u; max %u\n",
|
---|
903 | (unsigned)(cchNew / pThis->cPendingStrings), (unsigned)cchNewMin, (unsigned)cchNewMax);
|
---|
904 | }
|
---|
905 |
|
---|
906 | return true;
|
---|
907 | }
|
---|
908 |
|
---|
909 | #endif /* BLDPROG_STRTAB_WITH_COMPRESSION */
|
---|
910 |
|
---|
911 | /**
|
---|
912 | * Inserts a string into g_apUniqueStrings.
|
---|
913 | * @param pThis The strint table compiler instance.
|
---|
914 | * @param pStr The string.
|
---|
915 | */
|
---|
916 | static void bldProgStrTab_InsertUniqueString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
917 | {
|
---|
918 | size_t iIdx;
|
---|
919 | size_t iEnd = pThis->cSortedStrings;
|
---|
920 | if (iEnd)
|
---|
921 | {
|
---|
922 | size_t iStart = 0;
|
---|
923 | for (;;)
|
---|
924 | {
|
---|
925 | iIdx = iStart + (iEnd - iStart) / 2;
|
---|
926 | if (pThis->papSortedStrings[iIdx]->cchString < pStr->cchString)
|
---|
927 | {
|
---|
928 | if (iIdx <= iStart)
|
---|
929 | break;
|
---|
930 | iEnd = iIdx;
|
---|
931 | }
|
---|
932 | else if (pThis->papSortedStrings[iIdx]->cchString > pStr->cchString)
|
---|
933 | {
|
---|
934 | if (++iIdx >= iEnd)
|
---|
935 | break;
|
---|
936 | iStart = iIdx;
|
---|
937 | }
|
---|
938 | else
|
---|
939 | break;
|
---|
940 | }
|
---|
941 |
|
---|
942 | if (iIdx != pThis->cSortedStrings)
|
---|
943 | memmove(&pThis->papSortedStrings[iIdx + 1], &pThis->papSortedStrings[iIdx],
|
---|
944 | (pThis->cSortedStrings - iIdx) * sizeof(pThis->papSortedStrings[iIdx]));
|
---|
945 | }
|
---|
946 | else
|
---|
947 | iIdx = 0;
|
---|
948 |
|
---|
949 | pThis->papSortedStrings[iIdx] = pStr;
|
---|
950 | pThis->cSortedStrings++;
|
---|
951 | }
|
---|
952 |
|
---|
953 |
|
---|
954 | /**
|
---|
955 | * Compiles the string table after the string has been added.
|
---|
956 | *
|
---|
957 | * This will save space by dropping string terminators, eliminating duplicates
|
---|
958 | * and try find strings that are sub-strings of others.
|
---|
959 | *
|
---|
960 | * Will initialize the StrRef of all StrTabString instances.
|
---|
961 | *
|
---|
962 | * @returns success indicator (error flagged).
|
---|
963 | * @param pThis The strint table compiler instance.
|
---|
964 | */
|
---|
965 | static bool BldProgStrTab_CompileIt(PBLDPROGSTRTAB pThis, bool fVerbose)
|
---|
966 | {
|
---|
967 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
968 | /*
|
---|
969 | * Do the compression and add all the compressed strings to the table.
|
---|
970 | */
|
---|
971 | if (!bldProgStrTab_compressorDoStringCompression(pThis, fVerbose))
|
---|
972 | return false;
|
---|
973 |
|
---|
974 | /*
|
---|
975 | * Add the dictionary strings.
|
---|
976 | */
|
---|
977 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
978 | if (pThis->aCompDict[i].cchString > 1)
|
---|
979 | bldProgStrTab_AddStringToHashTab(pThis, &pThis->aCompDict[i]);
|
---|
980 | # ifdef RT_STRICT
|
---|
981 | else if (pThis->aCompDict[i].cchString != 1)
|
---|
982 | abort();
|
---|
983 | # endif
|
---|
984 | #endif
|
---|
985 | if (fVerbose)
|
---|
986 | printf("debug: %u unique strings (%u bytes), %u duplicates, %u collisions\n",
|
---|
987 | (unsigned)pThis->cUniqueStrings, (unsigned)pThis->cchUniqueStrings,
|
---|
988 | (unsigned)pThis->cDuplicateStrings, (unsigned)pThis->cCollisions);
|
---|
989 |
|
---|
990 | /*
|
---|
991 | * Create papSortedStrings from the hash table. The table is sorted by
|
---|
992 | * string length, with the longer strings first, so that we increase our
|
---|
993 | * chances of locating duplicate substrings.
|
---|
994 | */
|
---|
995 | pThis->papSortedStrings = (PBLDPROGSTRING *)malloc(sizeof(pThis->papSortedStrings[0]) * pThis->cUniqueStrings);
|
---|
996 | if (!pThis->papSortedStrings)
|
---|
997 | return false;
|
---|
998 | pThis->cSortedStrings = 0;
|
---|
999 | size_t idxHash = pThis->cStrHash;
|
---|
1000 | while (idxHash-- > 0)
|
---|
1001 | {
|
---|
1002 | PBLDPROGSTRING pCur = pThis->papStrHash[idxHash];
|
---|
1003 | if (pCur)
|
---|
1004 | {
|
---|
1005 | do
|
---|
1006 | {
|
---|
1007 | bldProgStrTab_InsertUniqueString(pThis, pCur);
|
---|
1008 | pCur = pCur->pNextCollision;
|
---|
1009 | } while (pCur);
|
---|
1010 | }
|
---|
1011 | }
|
---|
1012 |
|
---|
1013 | /*
|
---|
1014 | * Create the actual string table.
|
---|
1015 | */
|
---|
1016 | pThis->pachStrTab = (char *)malloc(pThis->cchUniqueStrings + 1);
|
---|
1017 | if (!pThis->pachStrTab)
|
---|
1018 | return false;
|
---|
1019 | pThis->cchStrTab = 0;
|
---|
1020 | for (size_t i = 0; i < pThis->cSortedStrings; i++)
|
---|
1021 | {
|
---|
1022 | PBLDPROGSTRING pCur = pThis->papSortedStrings[i];
|
---|
1023 | const char * const pszCur = pCur->pszString;
|
---|
1024 | size_t const cchCur = pCur->cchString;
|
---|
1025 | size_t offStrTab = pThis->cchStrTab;
|
---|
1026 |
|
---|
1027 | /*
|
---|
1028 | * See if the string is a substring already found in the string table.
|
---|
1029 | * Excluding the zero terminator increases the chances for this.
|
---|
1030 | */
|
---|
1031 | size_t cchLeft = pThis->cchStrTab >= cchCur ? pThis->cchStrTab - cchCur : 0;
|
---|
1032 | const char *pchLeft = pThis->pachStrTab;
|
---|
1033 | char const chFirst = *pszCur;
|
---|
1034 | while (cchLeft > 0)
|
---|
1035 | {
|
---|
1036 | const char *pchCandidate = (const char *)memchr(pchLeft, chFirst, cchLeft);
|
---|
1037 | if (!pchCandidate)
|
---|
1038 | break;
|
---|
1039 | if (memcmp(pchCandidate, pszCur, cchCur) == 0)
|
---|
1040 | {
|
---|
1041 | offStrTab = pchCandidate - pThis->pachStrTab;
|
---|
1042 | break;
|
---|
1043 | }
|
---|
1044 |
|
---|
1045 | cchLeft -= pchCandidate + 1 - pchLeft;
|
---|
1046 | pchLeft = pchCandidate + 1;
|
---|
1047 | }
|
---|
1048 |
|
---|
1049 | if (offStrTab == pThis->cchStrTab)
|
---|
1050 | {
|
---|
1051 | /*
|
---|
1052 | * See if the start of the string overlaps the end of the string table.
|
---|
1053 | */
|
---|
1054 | if (pThis->cchStrTab && cchCur > 1)
|
---|
1055 | {
|
---|
1056 | cchLeft = RT_MIN(pThis->cchStrTab, cchCur - 1);
|
---|
1057 | pchLeft = &pThis->pachStrTab[pThis->cchStrTab - cchLeft];
|
---|
1058 | while (cchLeft > 0)
|
---|
1059 | {
|
---|
1060 | const char *pchCandidate = (const char *)memchr(pchLeft, chFirst, cchLeft);
|
---|
1061 | if (!pchCandidate)
|
---|
1062 | break;
|
---|
1063 | cchLeft -= pchCandidate - pchLeft;
|
---|
1064 | pchLeft = pchCandidate;
|
---|
1065 | if (memcmp(pchLeft, pszCur, cchLeft) == 0)
|
---|
1066 | {
|
---|
1067 | size_t cchToCopy = cchCur - cchLeft;
|
---|
1068 | memcpy(&pThis->pachStrTab[offStrTab], &pszCur[cchLeft], cchToCopy);
|
---|
1069 | pThis->cchStrTab += cchToCopy;
|
---|
1070 | offStrTab = pchCandidate - pThis->pachStrTab;
|
---|
1071 | break;
|
---|
1072 | }
|
---|
1073 | cchLeft--;
|
---|
1074 | pchLeft++;
|
---|
1075 | }
|
---|
1076 | }
|
---|
1077 |
|
---|
1078 | /*
|
---|
1079 | * If we didn't have any luck above, just append the string.
|
---|
1080 | */
|
---|
1081 | if (offStrTab == pThis->cchStrTab)
|
---|
1082 | {
|
---|
1083 | memcpy(&pThis->pachStrTab[offStrTab], pszCur, cchCur);
|
---|
1084 | pThis->cchStrTab += cchCur;
|
---|
1085 | }
|
---|
1086 | }
|
---|
1087 |
|
---|
1088 | /*
|
---|
1089 | * Set the string table offset for all the references to this string.
|
---|
1090 | */
|
---|
1091 | do
|
---|
1092 | {
|
---|
1093 | pCur->offStrTab = (uint32_t)offStrTab;
|
---|
1094 | pCur = pCur->pNextRef;
|
---|
1095 | } while (pCur != NULL);
|
---|
1096 | }
|
---|
1097 |
|
---|
1098 | if (fVerbose)
|
---|
1099 | printf("debug: String table: %u bytes\n", (unsigned)pThis->cchStrTab);
|
---|
1100 | return true;
|
---|
1101 | }
|
---|
1102 |
|
---|
1103 |
|
---|
1104 | #ifdef RT_STRICT
|
---|
1105 | /**
|
---|
1106 | * Sanity checks a string table string.
|
---|
1107 | * @param pThis The strint table compiler instance.
|
---|
1108 | * @param pStr The string to check.
|
---|
1109 | */
|
---|
1110 | static void BldProgStrTab_CheckStrTabString(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pStr)
|
---|
1111 | {
|
---|
1112 | if (pStr->cchString != strlen(pStr->pszString))
|
---|
1113 | abort();
|
---|
1114 | if (pStr->offStrTab >= pThis->cchStrTab)
|
---|
1115 | abort();
|
---|
1116 | if (pStr->offStrTab + pStr->cchString > pThis->cchStrTab)
|
---|
1117 | abort();
|
---|
1118 | if (memcmp(pStr->pszString, &pThis->pachStrTab[pStr->offStrTab], pStr->cchString) != 0)
|
---|
1119 | abort();
|
---|
1120 | }
|
---|
1121 | #endif
|
---|
1122 |
|
---|
1123 |
|
---|
1124 | /**
|
---|
1125 | * Output the string table string in C string litteral fashion.
|
---|
1126 | *
|
---|
1127 | * @param pThis The strint table instance.
|
---|
1128 | * @param pString The string to print.
|
---|
1129 | * @param pOut The output stream.
|
---|
1130 | */
|
---|
1131 | static void BldProgStrTab_PrintCStringLitteral(PBLDPROGSTRTAB pThis, PBLDPROGSTRING pString, FILE *pOut)
|
---|
1132 | {
|
---|
1133 | const unsigned char *psz = (const unsigned char *)pString->pszString;
|
---|
1134 | unsigned char uch;
|
---|
1135 | while ((uch = *psz++) != '\0')
|
---|
1136 | {
|
---|
1137 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1138 | if (pThis->aCompDict[uch].cchString == 1)
|
---|
1139 | #else
|
---|
1140 | if (!(uch & 0x80))
|
---|
1141 | #endif
|
---|
1142 | {
|
---|
1143 | if (uch != '\'' && uch != '\\')
|
---|
1144 | fputc((char)uch, pOut);
|
---|
1145 | else
|
---|
1146 | {
|
---|
1147 | fputc('\\', pOut);
|
---|
1148 | fputc((char)uch, pOut);
|
---|
1149 | }
|
---|
1150 | }
|
---|
1151 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1152 | # ifndef BLDPROG_STRTAB_PURE_ASCII
|
---|
1153 | else if (uch == 0xff)
|
---|
1154 | {
|
---|
1155 | RTUNICP uc = RTStrGetCp((const char *)psz);
|
---|
1156 | psz += RTStrCpSize(uc);
|
---|
1157 | fprintf(pOut, "\\u%04x", uc);
|
---|
1158 | }
|
---|
1159 | # else
|
---|
1160 | else
|
---|
1161 | fputs(pThis->aCompDict[uch].pszString, pOut);
|
---|
1162 | # endif
|
---|
1163 | #else
|
---|
1164 | else
|
---|
1165 | fprintf(pOut, "\\x%02x", (unsigned)uch);
|
---|
1166 | NOREF(pThis);
|
---|
1167 | #endif
|
---|
1168 | }
|
---|
1169 | }
|
---|
1170 |
|
---|
1171 |
|
---|
1172 | /**
|
---|
1173 | * Writes the string table code to the output stream.
|
---|
1174 | *
|
---|
1175 | * @param pThis The strint table compiler instance.
|
---|
1176 | * @param pOut The output stream.
|
---|
1177 | * @param pszScope The scoping ("static " or empty string),
|
---|
1178 | * including trailing space.
|
---|
1179 | * @param pszPrefix The variable prefix. Typically "g_" for C programs,
|
---|
1180 | * whereas C++ classes normally use "class::s_g".
|
---|
1181 | * @param pszBaseName The base name for the variables. The user
|
---|
1182 | * accessible variable will end up as just the
|
---|
1183 | * prefix and basename concatenated.
|
---|
1184 | */
|
---|
1185 | static void BldProgStrTab_WriteStringTable(PBLDPROGSTRTAB pThis, FILE *pOut,
|
---|
1186 | const char *pszScope, const char *pszPrefix, const char *pszBaseName)
|
---|
1187 | {
|
---|
1188 | #ifdef RT_STRICT
|
---|
1189 | /*
|
---|
1190 | * Do some quick sanity checks while we're here.
|
---|
1191 | */
|
---|
1192 | # ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1193 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
1194 | {
|
---|
1195 | if (BldProgBitIsSet(pThis->bmUsedChars, i)
|
---|
1196 | ? pThis->aCompDict[i].cchString != 1 : pThis->aCompDict[i].cchString < 1)
|
---|
1197 | abort();
|
---|
1198 | if (pThis->aCompDict[i].cchString > 1)
|
---|
1199 | BldProgStrTab_CheckStrTabString(pThis, &pThis->aCompDict[i]);
|
---|
1200 | }
|
---|
1201 | # endif
|
---|
1202 | #endif
|
---|
1203 |
|
---|
1204 | /*
|
---|
1205 | * Create a table for speeding up the character categorization.
|
---|
1206 | */
|
---|
1207 | uint8_t abCharCat[256];
|
---|
1208 | memset(abCharCat, 0, sizeof(abCharCat));
|
---|
1209 | abCharCat[(unsigned char)'\\'] = 1;
|
---|
1210 | abCharCat[(unsigned char)'\''] = 1;
|
---|
1211 | for (unsigned i = 0; i < 0x20; i++)
|
---|
1212 | abCharCat[i] = 2;
|
---|
1213 | for (unsigned i = 0x7f; i < 0x100; i++)
|
---|
1214 | abCharCat[i] = 2;
|
---|
1215 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1216 | for (unsigned i = 0; i < 0x100; i++)
|
---|
1217 | if (!BldProgBitIsSet(pThis->bmUsedChars, i)) /* Encode table references using '\xYY'. */
|
---|
1218 | abCharCat[i] = 2;
|
---|
1219 | #endif
|
---|
1220 |
|
---|
1221 | /*
|
---|
1222 | * We follow the sorted string table, one string per line.
|
---|
1223 | */
|
---|
1224 | fprintf(pOut,
|
---|
1225 | "#include <iprt/bldprog-strtab.h>\n"
|
---|
1226 | "\n"
|
---|
1227 | "static const char g_achStrTab%s[] =\n"
|
---|
1228 | "{\n",
|
---|
1229 | pszBaseName);
|
---|
1230 |
|
---|
1231 | uint32_t off = 0;
|
---|
1232 | for (uint32_t i = 0; i < pThis->cSortedStrings; i++)
|
---|
1233 | {
|
---|
1234 | PBLDPROGSTRING pCur = pThis->papSortedStrings[i];
|
---|
1235 | uint32_t offEnd = pCur->offStrTab + (uint32_t)pCur->cchString;
|
---|
1236 | if (offEnd > off)
|
---|
1237 | {
|
---|
1238 | /* Comment with an uncompressed and more readable version of the string. */
|
---|
1239 | if (off == pCur->offStrTab)
|
---|
1240 | fprintf(pOut, "/* 0x%05x = \"", off);
|
---|
1241 | else
|
---|
1242 | fprintf(pOut, "/* 0X%05x = \"", off);
|
---|
1243 | BldProgStrTab_PrintCStringLitteral(pThis, pCur, pOut);
|
---|
1244 | fputs("\" */\n", pOut);
|
---|
1245 |
|
---|
1246 | /* Must use char by char here or we may trigger the max string
|
---|
1247 | length limit in the compiler, */
|
---|
1248 | fputs(" ", pOut);
|
---|
1249 | for (; off < offEnd; off++)
|
---|
1250 | {
|
---|
1251 | unsigned char uch = pThis->pachStrTab[off];
|
---|
1252 | fputc('\'', pOut);
|
---|
1253 | if (abCharCat[uch] == 0)
|
---|
1254 | fputc(uch, pOut);
|
---|
1255 | else if (abCharCat[uch] != 1)
|
---|
1256 | fprintf(pOut, "\\x%02x", (unsigned)uch);
|
---|
1257 | else
|
---|
1258 | {
|
---|
1259 | fputc('\\', pOut);
|
---|
1260 | fputc(uch, pOut);
|
---|
1261 | }
|
---|
1262 | fputc('\'', pOut);
|
---|
1263 | fputc(',', pOut);
|
---|
1264 | }
|
---|
1265 | fputc('\n', pOut);
|
---|
1266 | }
|
---|
1267 | }
|
---|
1268 |
|
---|
1269 | fprintf(pOut,
|
---|
1270 | "};\n"
|
---|
1271 | "AssertCompile(sizeof(g_achStrTab%s) == %#x);\n\n",
|
---|
1272 | pszBaseName, (unsigned)pThis->cchStrTab);
|
---|
1273 |
|
---|
1274 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1275 | /*
|
---|
1276 | * Write the compression dictionary.
|
---|
1277 | */
|
---|
1278 | fprintf(pOut,
|
---|
1279 | "static const RTBLDPROGSTRREF g_aCompDict%s[%u] = \n"
|
---|
1280 | "{\n",
|
---|
1281 | pszBaseName, (unsigned)RT_ELEMENTS(pThis->aCompDict));
|
---|
1282 | for (unsigned i = 0; i < RT_ELEMENTS(pThis->aCompDict); i++)
|
---|
1283 | if (pThis->aCompDict[i].cchString > 1)
|
---|
1284 | fprintf(pOut, " /*[%3u]=*/ { %#08x, %#04x }, // %6lu - %s\n", i,
|
---|
1285 | pThis->aCompDict[i].offStrTab, (unsigned)pThis->aCompDict[i].cchString,
|
---|
1286 | (unsigned long)pThis->auCompDictFreq[i], pThis->aCompDict[i].pszString);
|
---|
1287 | # ifndef BLDPROG_STRTAB_PURE_ASCII
|
---|
1288 | else if (i == 0xff)
|
---|
1289 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // UTF-8 escape\n", i);
|
---|
1290 | # endif
|
---|
1291 | else if (i == 0)
|
---|
1292 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // unused, because zero terminator\n", i);
|
---|
1293 | else if (i < 0x20)
|
---|
1294 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // %02x\n", i, i);
|
---|
1295 | else
|
---|
1296 | fprintf(pOut, " /*[%3u]=*/ { 0x000000, 0x00 }, // '%c'\n", i, (char)i);
|
---|
1297 | fprintf(pOut, "};\n\n");
|
---|
1298 | #endif
|
---|
1299 |
|
---|
1300 |
|
---|
1301 | /*
|
---|
1302 | * Write the string table data structure.
|
---|
1303 | */
|
---|
1304 | fprintf(pOut,
|
---|
1305 | "%sconst RTBLDPROGSTRTAB %s%s = \n"
|
---|
1306 | "{\n"
|
---|
1307 | " /*.pchStrTab = */ &g_achStrTab%s[0],\n"
|
---|
1308 | " /*.cchStrTab = */ sizeof(g_achStrTab%s),\n"
|
---|
1309 | ,
|
---|
1310 | pszScope, pszPrefix, pszBaseName, pszBaseName, pszBaseName);
|
---|
1311 | #ifdef BLDPROG_STRTAB_WITH_COMPRESSION
|
---|
1312 | fprintf(pOut,
|
---|
1313 | " /*.cCompDict = */ %u,\n"
|
---|
1314 | " /*.paCompDict = */ &g_aCompDict%s[0]\n"
|
---|
1315 | "};\n"
|
---|
1316 | # ifndef BLDPROG_STRTAB_PURE_ASCII /* 255 or 256 entries is how the decoder knows */
|
---|
1317 | , (unsigned)RT_ELEMENTS(pThis->aCompDict) - 1,
|
---|
1318 | # else
|
---|
1319 | , (unsigned)RT_ELEMENTS(pThis->aCompDict),
|
---|
1320 | # endif
|
---|
1321 | pszBaseName);
|
---|
1322 | #else
|
---|
1323 | fprintf(pOut,
|
---|
1324 | " /*.cCompDict = */ 0,\n"
|
---|
1325 | " /*.paCompDict = */ NULL\n"
|
---|
1326 | "};\n");
|
---|
1327 | #endif
|
---|
1328 | }
|
---|
1329 |
|
---|
1330 | #if RT_CLANG_PREREQ(4, 0) || RT_GNUC_PREREQ(4, 6)
|
---|
1331 | # pragma GCC diagnostic pop
|
---|
1332 | #endif
|
---|
1333 |
|
---|
1334 | #endif /* __cplusplus && IN_RING3 */
|
---|
1335 |
|
---|