From 40f9d989df2c1ff2f567656ccbdbfc0d97e34a77 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Sat, 13 Dec 2025 18:26:51 +0000 Subject: Switch cmeta from chibicc to a new homegrown lexer This lexer is being written as a chunklet, not quite quite complete yet as it lacks some functionality to make it generally useful for things, but good enough for the gluegen use case now. So it's in the repo now and we can go ahead and use it for this instead of having this hacked-to-pieces third party thing that's nowhere near as efficient. In the future the goal is to have a decently usable library for any sort of C metaprogramming needs, not just within this project. This design does the data-oriented thing of storing as little about each token as possible (just 5 bytes), and re-lexing specific pieces only when necessary. In some cases, full validation of correct syntax is only possible through this secondary step. Since the use case is metaprogramming and code-generation rather than the reimplementation of Clang, this slight sloppiness in validation doesn't seem too bad. It's been a while since I did a proper performance measurement of this code but in some crude tests I did in the past the primary tokenisation step was running at well over 500MB/s, which is fast enough for me. It's possible that this version is a tiny bit slower due to the added complexity of making UCNs work. UCNs, incidentally, are one of the dumbest features of C by far. However, unlike trigraphs - which this lexer does not handle - UCNs are still in the language, so it's kind of sort of necessary to support them. It's probably still possible to come up with a faster design with some SIMD trickery, but the main loop is currently branchless and the lookup tables are too big for the SIMD lookup things, so that seems kind of hard. A separate SIMD path for whitespace or comment runs seems dubious as it would introduce branch mispredictions everywhere. I also made previous attempts to unroll the main loop and every attempt just made it slower, so I guess code size is a significant factor. Optimising the secondary tokenisation of identifiers (and later numerals and string/character literals once those are handled) is still on the cards, but since that happens less often, I don't know how much difference it'll make. At any rate, in a multithreaded context this thing would already come pretty close to SSD speeds, if open-read-close syscall overhead doesn't get in the way first. I imagine it's fast enough for anyone who hasn't *already* written something faster. --- src/3p/chibicc/hashmap.c | 137 ----------------------------------------------- 1 file changed, 137 deletions(-) delete mode 100644 src/3p/chibicc/hashmap.c (limited to 'src/3p/chibicc/hashmap.c') diff --git a/src/3p/chibicc/hashmap.c b/src/3p/chibicc/hashmap.c deleted file mode 100644 index 2090274..0000000 --- a/src/3p/chibicc/hashmap.c +++ /dev/null @@ -1,137 +0,0 @@ -// This is an implementation of the open-addressing hash table. - -#include "chibicc.h" - -// mike: moved from chibicc.h and also renamed, to avoid conflicts with langext.h -#define chibi_unreachable() error("internal error at %s:%d", __FILE__, __LINE__) - -// Initial hash bucket size -#define INIT_SIZE 16 - -// Rehash if the usage exceeds 70%. -#define HIGH_WATERMARK 70 - -// We'll keep the usage below 50% after rehashing. -#define LOW_WATERMARK 50 - -// Represents a deleted hash entry -#define TOMBSTONE ((void *)-1) - -static uint64_t fnv_hash(char *s, int len) { - uint64_t hash = 0xcbf29ce484222325; - for (int i = 0; i < len; i++) { - hash *= 0x100000001b3; - hash ^= (unsigned char)s[i]; - } - return hash; -} - -// Make room for new entires in a given hashmap by removing -// tombstones and possibly extending the bucket size. -static void rehash(HashMap *map) { - // Compute the size of the new hashmap. - int nkeys = 0; - for (int i = 0; i < map->capacity; i++) - if (map->buckets[i].key && map->buckets[i].key != TOMBSTONE) - nkeys++; - - int cap = map->capacity; - while ((nkeys * 100) / cap >= LOW_WATERMARK) - cap = cap * 2; - assert(cap > 0); - - // Create a new hashmap and copy all key-values. - HashMap map2 = {0}; - map2.buckets = calloc(cap, sizeof(HashEntry)); - map2.capacity = cap; - - for (int i = 0; i < map->capacity; i++) { - HashEntry *ent = &map->buckets[i]; - if (ent->key && ent->key != TOMBSTONE) - hashmap_put2(&map2, ent->key, ent->keylen, ent->val); - } - - assert(map2.used == nkeys); - *map = map2; -} - -static bool match(HashEntry *ent, char *key, int keylen) { - return ent->key && ent->key != TOMBSTONE && - ent->keylen == keylen && memcmp(ent->key, key, keylen) == 0; -} - -static HashEntry *get_entry(HashMap *map, char *key, int keylen) { - if (!map->buckets) - return NULL; - - uint64_t hash = fnv_hash(key, keylen); - - for (int i = 0; i < map->capacity; i++) { - HashEntry *ent = &map->buckets[(hash + i) % map->capacity]; - if (match(ent, key, keylen)) - return ent; - if (ent->key == NULL) - return NULL; - } - chibi_unreachable(); -} - -static HashEntry *get_or_insert_entry(HashMap *map, char *key, int keylen) { - if (!map->buckets) { - map->buckets = calloc(INIT_SIZE, sizeof(HashEntry)); - map->capacity = INIT_SIZE; - } else if ((map->used * 100) / map->capacity >= HIGH_WATERMARK) { - rehash(map); - } - - uint64_t hash = fnv_hash(key, keylen); - - for (int i = 0; i < map->capacity; i++) { - HashEntry *ent = &map->buckets[(hash + i) % map->capacity]; - - if (match(ent, key, keylen)) - return ent; - - if (ent->key == TOMBSTONE) { - ent->key = key; - ent->keylen = keylen; - return ent; - } - - if (ent->key == NULL) { - ent->key = key; - ent->keylen = keylen; - map->used++; - return ent; - } - } - chibi_unreachable(); -} - -void *hashmap_get(HashMap *map, char *key) { - return hashmap_get2(map, key, strlen(key)); -} - -void *hashmap_get2(HashMap *map, char *key, int keylen) { - HashEntry *ent = get_entry(map, key, keylen); - return ent ? ent->val : NULL; -} - -void hashmap_put(HashMap *map, char *key, void *val) { - hashmap_put2(map, key, strlen(key), val); -} - -void hashmap_put2(HashMap *map, char *key, int keylen, void *val) { - HashEntry *ent = get_or_insert_entry(map, key, keylen); - ent->val = val; -} - -void hashmap_delete(HashMap *map, char *key) { - hashmap_delete2(map, key, strlen(key)); -} - -void hashmap_delete2(HashMap *map, char *key, int keylen) { - HashEntry *ent = get_entry(map, key, keylen); - if (ent) - ent->key = TOMBSTONE; -} -- cgit v1.2.3-54-g00ecf