Switch cmeta from chibicc to a new homegrown lexer

This lexer is being written as a chunklet, not quite quite complete yet
as it lacks some functionality to make it generally useful for things,
but good enough for the gluegen use case now. So it's in the repo now
and we can go ahead and use it for this instead of having this
hacked-to-pieces third party thing that's nowhere near as efficient. In
the future the goal is to have a decently usable library for any sort of
C metaprogramming needs, not just within this project.

This design does the data-oriented thing of storing as little about each
token as possible (just 5 bytes), and re-lexing specific pieces only
when necessary. In some cases, full validation of correct syntax is
only possible through this secondary step. Since the use case is
metaprogramming and code-generation rather than the reimplementation of
Clang, this slight sloppiness in validation doesn't seem too bad.

It's been a while since I did a proper performance measurement of this
code but in some crude tests I did in the past the primary tokenisation
step was running at well over 500MB/s, which is fast enough for me. It's
possible that this version is a tiny bit slower due to the added
complexity of making UCNs work. UCNs, incidentally, are one of the
dumbest features of C by far. However, unlike trigraphs - which this
lexer does not handle - UCNs are still in the language, so it's kind of
sort of necessary to support them.

It's probably still possible to come up with a faster design with some
SIMD trickery, but the main loop is currently branchless and the lookup
tables are too big for the SIMD lookup things, so that seems kind of
hard. A separate SIMD path for whitespace or comment runs seems dubious
as it would introduce branch mispredictions everywhere. I also made
previous attempts to unroll the main loop and every attempt just made it
slower, so I guess code size is a significant factor.

Optimising the secondary tokenisation of identifiers (and later numerals
and string/character literals once those are handled) is still on the
cards, but since that happens less often, I don't know how much
difference it'll make.

At any rate, in a multithreaded context this thing would already come
pretty close to SSD speeds, if open-read-close syscall overhead doesn't
get in the way first. I imagine it's fast enough for anyone who hasn't
*already* written something faster.

1 files changed, 0 insertions, 785 deletions

diff --git a/src/3p/chibicc/tokenize.c b/src/3p/chibicc/tokenize.c
deleted file mode 100644
index 6738206..0000000
--- a/src/3p/chibicc/tokenize.c
+++ /dev/null
@@ -1,785 +0,0 @@
-#include "chibicc.h"
-
-#ifdef _WIN32
-#define strncasecmp _strnicmp
-#endif
-
-// Input file
-static File *current_file;
-
-// A list of all input files.
-static File **input_files;
-
-// True if the current position is at the beginning of a line
-static bool at_bol;
-
-// True if the current position follows a space character
-static bool has_space;
-
-// Reports an error and exit.
-void error(char *fmt, ...) {
-  va_list ap;
-  va_start(ap, fmt);
-  fprintf(stderr, "cmeta: chibicc: ");
-  vfprintf(stderr, fmt, ap);
-  fprintf(stderr, "\n");
-  exit(1);
-}
-
-// Reports an error message in the following format.
-//
-// foo.c:10: x = y + 1;
-//               ^ <error message here>
-static void verror_at(char *filename, char *input, int line_no,
-                      char *loc, char *fmt, va_list ap) {
-  // Find a line containing `loc`.
-  char *line = loc;
-  while (input < line && line[-1] != '\n')
-    line--;
-
-  char *end = loc;
-  while (*end && *end != '\n')
-    end++;
-
-  // Print out the line.
-  int indent = fprintf(stderr, "%s:%d: ", filename, line_no);
-  fprintf(stderr, "%.*s\n", (int)(end - line), line);
-
-  // Show the error message.
-  int pos = display_width(line, loc - line) + indent;
-
-  fprintf(stderr, "%*s", pos, ""); // print pos spaces.
-  fprintf(stderr, "^ ");
-  vfprintf(stderr, fmt, ap);
-  fprintf(stderr, "\n");
-}
-
-void error_at(char *loc, char *fmt, ...) {
-  int line_no = 1;
-  for (char *p = current_file->contents; p < loc; p++)
-    if (*p == '\n')
-      line_no++;
-
-  va_list ap;
-  va_start(ap, fmt);
-  verror_at(current_file->name, current_file->contents, line_no, loc, fmt, ap);
-  exit(1);
-}
-
-void error_tok(Token *tok, char *fmt, ...) {
-  va_list ap;
-  va_start(ap, fmt);
-  verror_at(tok->file->name, tok->file->contents, tok->line_no, tok->loc, fmt, ap);
-  exit(1);
-}
-
-void warn_tok(Token *tok, char *fmt, ...) {
-  va_list ap;
-  va_start(ap, fmt);
-  verror_at(tok->file->name, tok->file->contents, tok->line_no, tok->loc, fmt, ap);
-  va_end(ap);
-}
-
-// Consumes the current token if it matches `op`.
-bool equal(const Token *tok, const char *op) {
-  return memcmp(tok->loc, op, tok->len) == 0 && op[tok->len] == '\0';
-}
-
-// Ensure that the current token is `op`.
-Token *skip(Token *tok, char *op) {
-  if (!equal(tok, op))
-    error_tok(tok, "expected '%s'", op);
-  return tok->next;
-}
-
-bool consume(Token **rest, Token *tok, char *str) {
-  if (equal(tok, str)) {
-    *rest = tok->next;
-    return true;
-  }
-  *rest = tok;
-  return false;
-}
-
-// Create a new token.
-static Token *new_token(TokenKind kind, char *start, char *end) {
-  Token *tok = calloc(1, sizeof(Token));
-  tok->kind = kind;
-  tok->loc = start;
-  tok->len = end - start;
-  tok->file = current_file;
-  tok->filename = current_file->display_name;
-  tok->at_bol = at_bol;
-  tok->has_space = has_space;
-
-  at_bol = has_space = false;
-  return tok;
-}
-
-static bool startswith(char *p, char *q) {
-  return strncmp(p, q, strlen(q)) == 0;
-}
-
-// Read an identifier and returns the length of it.
-// If p does not point to a valid identifier, 0 is returned.
-static int read_ident(char *start) {
-  char *p = start;
-  uint32_t c = decode_utf8(&p, p);
-  if (!is_ident1(c))
-    return 0;
-
-  for (;;) {
-    char *q;
-    c = decode_utf8(&q, p);
-    if (!is_ident2(c))
-      return p - start;
-    p = q;
-  }
-}
-
-static int from_hex(char c) {
-  if ('0' <= c && c <= '9')
-    return c - '0';
-  if ('a' <= c && c <= 'f')
-    return c - 'a' + 10;
-  return c - 'A' + 10;
-}
-
-// Read a punctuator token from p and returns its length.
-static int read_punct(char *p) {
-  static char *kw[] = {
-    "<<=", ">>=", "...", "==", "!=", "<=", ">=", "->", "+=",
-    "-=", "*=", "/=", "++", "--", "%=", "&=", "|=", "^=", "&&",
-    "||", "<<", ">>", "##",
-  };
-
-  for (int i = 0; i < sizeof(kw) / sizeof(*kw); i++)
-    if (startswith(p, kw[i]))
-      return strlen(kw[i]);
-
-  return ispunct(*p) ? 1 : 0;
-}
-
-static bool is_keyword(Token *tok) {
-  static HashMap map;
-
-  if (map.capacity == 0) {
-    static char *kw[] = {
-      "return", "if", "else", "for", "while", "int", "sizeof", "char",
-      "struct", "union", "short", "long", "void", "typedef", "_Bool",
-      "enum", "static", "goto", "break", "continue", "switch", "case",
-      "default", "extern", "_Alignof", "_Alignas", "do", "signed",
-      "unsigned", "const", "volatile", "auto", "register", "restrict",
-      "__restrict", "__restrict__", "_Noreturn", "float", "double",
-      "typeof", "asm", "_Thread_local", "__thread", "_Atomic",
-      "__attribute__",
-    };
-
-    for (int i = 0; i < sizeof(kw) / sizeof(*kw); i++)
-      hashmap_put(&map, kw[i], (void *)1);
-  }
-
-  return hashmap_get2(&map, tok->loc, tok->len);
-}
-
-static int read_escaped_char(char **new_pos, char *p) {
-  if ('0' <= *p && *p <= '7') {
-    // Read an octal number.
-    int c = *p++ - '0';
-    if ('0' <= *p && *p <= '7') {
-      c = (c << 3) + (*p++ - '0');
-      if ('0' <= *p && *p <= '7')
-        c = (c << 3) + (*p++ - '0');
-    }
-    *new_pos = p;
-    return c;
-  }
-
-  if (*p == 'x') {
-    // Read a hexadecimal number.
-    p++;
-    if (!isxdigit(*p))
-      error_at(p, "invalid hex escape sequence");
-
-    int c = 0;
-    for (; isxdigit(*p); p++)
-      c = (c << 4) + from_hex(*p);
-    *new_pos = p;
-    return c;
-  }
-
-  *new_pos = p + 1;
-
-  // Escape sequences are defined using themselves here. E.g.
-  // '\n' is implemented using '\n'. This tautological definition
-  // works because the compiler that compiles our compiler knows
-  // what '\n' actually is. In other words, we "inherit" the ASCII
-  // code of '\n' from the compiler that compiles our compiler,
-  // so we don't have to teach the actual code here.
-  //
-  // This fact has huge implications not only for the correctness
-  // of the compiler but also for the security of the generated code.
-  // For more info, read "Reflections on Trusting Trust" by Ken Thompson.
-  // https://github.com/rui314/chibicc/wiki/thompson1984.pdf
-  switch (*p) {
-  case 'a': return '\a';
-  case 'b': return '\b';
-  case 't': return '\t';
-  case 'n': return '\n';
-  case 'v': return '\v';
-  case 'f': return '\f';
-  case 'r': return '\r';
-  // [GNU] \e for the ASCII escape character is a GNU C extension.
-  case 'e': return 27;
-  default: return *p;
-  }
-}
-
-// Find a closing double-quote.
-static char *string_literal_end(char *p) {
-  char *start = p;
-  for (; *p != '"'; p++) {
-    if (*p == '\n' || *p == '\0')
-      error_at(start, "unclosed string literal");
-    if (*p == '\\')
-      p++;
-  }
-  return p;
-}
-
-static Token *read_string_literal(char *start, char *quote) {
-  char *end = string_literal_end(quote + 1);
-  char *buf = calloc(1, end - quote);
-  int len = 0;
-
-  for (char *p = quote + 1; p < end;) {
-    if (*p == '\\')
-      buf[len++] = read_escaped_char(&p, p + 1);
-    else
-      buf[len++] = *p++;
-  }
-
-  Token *tok = new_token(TK_STR, start, end + 1);
-  tok->ty = array_of(ty_char, len + 1);
-  tok->str = buf;
-  return tok;
-}
-
-// Read a UTF-8-encoded string literal and transcode it in UTF-16.
-//
-// UTF-16 is yet another variable-width encoding for Unicode. Code
-// points smaller than U+10000 are encoded in 2 bytes. Code points
-// equal to or larger than that are encoded in 4 bytes. Each 2 bytes
-// in the 4 byte sequence is called "surrogate", and a 4 byte sequence
-// is called a "surrogate pair".
-static Token *read_utf16_string_literal(char *start, char *quote) {
-  char *end = string_literal_end(quote + 1);
-  uint16_t *buf = calloc(2, end - start);
-  int len = 0;
-
-  for (char *p = quote + 1; p < end;) {
-    if (*p == '\\') {
-      buf[len++] = read_escaped_char(&p, p + 1);
-      continue;
-    }
-
-    uint32_t c = decode_utf8(&p, p);
-    if (c < 0x10000) {
-      // Encode a code point in 2 bytes.
-      buf[len++] = c;
-    } else {
-      // Encode a code point in 4 bytes.
-      c -= 0x10000;
-      buf[len++] = 0xd800 + ((c >> 10) & 0x3ff);
-      buf[len++] = 0xdc00 + (c & 0x3ff);
-    }
-  }
-
-  Token *tok = new_token(TK_STR, start, end + 1);
-  tok->ty = array_of(ty_ushort, len + 1);
-  tok->str = (char *)buf;
-  return tok;
-}
-
-// Read a UTF-8-encoded string literal and transcode it in UTF-32.
-//
-// UTF-32 is a fixed-width encoding for Unicode. Each code point is
-// encoded in 4 bytes.
-static Token *read_utf32_string_literal(char *start, char *quote, Type *ty) {
-  char *end = string_literal_end(quote + 1);
-  uint32_t *buf = calloc(4, end - quote);
-  int len = 0;
-
-  for (char *p = quote + 1; p < end;) {
-    if (*p == '\\')
-      buf[len++] = read_escaped_char(&p, p + 1);
-    else
-      buf[len++] = decode_utf8(&p, p);
-  }
-
-  Token *tok = new_token(TK_STR, start, end + 1);
-  tok->ty = array_of(ty, len + 1);
-  tok->str = (char *)buf;
-  return tok;
-}
-
-static Token *read_char_literal(char *start, char *quote, Type *ty) {
-  char *p = quote + 1;
-  if (*p == '\0')
-    error_at(start, "unclosed char literal");
-
-  int c;
-  if (*p == '\\')
-    c = read_escaped_char(&p, p + 1);
-  else
-    c = decode_utf8(&p, p);
-
-  char *end = p;
-  for (; *end != '\''; ++end) {
-    if (!*end || *end == '\n')
-      error_at(p, "unclosed char literal");
-  }
-
-  Token *tok = new_token(TK_NUM, start, end + 1);
-  tok->val = c;
-  tok->ty = ty;
-  return tok;
-}
-
-static bool convert_pp_int(Token *tok) {
-  char *p = tok->loc;
-
-  // Read a binary, octal, decimal or hexadecimal number.
-  int base = 10;
-  if (!strncasecmp(p, "0x", 2) && isxdigit(p[2])) {
-    p += 2;
-    base = 16;
-  } else if (!strncasecmp(p, "0b", 2) && (p[2] == '0' || p[2] == '1')) {
-    p += 2;
-    base = 2;
-  } else if (*p == '0') {
-    base = 8;
-  }
-
-  int64_t val = strtoul(p, &p, base);
-
-  // Read U, L or LL suffixes.
-  bool l = false;
-  bool u = false;
-
-  if (startswith(p, "LLU") || startswith(p, "LLu") ||
-      startswith(p, "llU") || startswith(p, "llu") ||
-      startswith(p, "ULL") || startswith(p, "Ull") ||
-      startswith(p, "uLL") || startswith(p, "ull")) {
-    p += 3;
-    l = u = true;
-  } else if (!strncasecmp(p, "lu", 2) || !strncasecmp(p, "ul", 2)) {
-    p += 2;
-    l = u = true;
-  } else if (startswith(p, "LL") || startswith(p, "ll")) {
-    p += 2;
-    l = true;
-  } else if (*p == 'L' || *p == 'l') {
-    p++;
-    l = true;
-  } else if (*p == 'U' || *p == 'u') {
-    p++;
-    u = true;
-  }
-
-  if (p != tok->loc + tok->len)
-    return false;
-
-  // Infer a type.
-  Type *ty;
-  if (base == 10) {
-    if (l && u)
-      ty = ty_ulong;
-    else if (l)
-      ty = ty_long;
-    else if (u)
-      ty = (val >> 32) ? ty_ulong : ty_uint;
-    else
-      ty = (val >> 31) ? ty_long : ty_int;
-  } else {
-    if (l && u)
-      ty = ty_ulong;
-    else if (l)
-      ty = (val >> 63) ? ty_ulong : ty_long;
-    else if (u)
-      ty = (val >> 32) ? ty_ulong : ty_uint;
-    else if (val >> 63)
-      ty = ty_ulong;
-    else if (val >> 32)
-      ty = ty_long;
-    else if (val >> 31)
-      ty = ty_uint;
-    else
-      ty = ty_int;
-  }
-
-  tok->kind = TK_NUM;
-  tok->val = val;
-  tok->ty = ty;
-  return true;
-}
-
-// The definition of the numeric literal at the preprocessing stage
-// is more relaxed than the definition of that at the later stages.
-// In order to handle that, a numeric literal is tokenized as a
-// "pp-number" token first and then converted to a regular number
-// token after preprocessing.
-//
-// This function converts a pp-number token to a regular number token.
-static void convert_pp_number(Token *tok) {
-  // Try to parse as an integer constant.
-  if (convert_pp_int(tok))
-    return;
-
-  // If it's not an integer, it must be a floating point constant.
-  char *end;
-  long double val = strtold(tok->loc, &end);
-
-  Type *ty;
-  if (*end == 'f' || *end == 'F') {
-    ty = ty_float;
-    end++;
-  } else if (*end == 'l' || *end == 'L') {
-    ty = ty_ldouble;
-    end++;
-  } else {
-    ty = ty_double;
-  }
-
-  if (tok->loc + tok->len != end)
-    error_tok(tok, "invalid numeric constant");
-
-  tok->kind = TK_NUM;
-  tok->fval = val;
-  tok->ty = ty;
-}
-
-void convert_pp_tokens(Token *tok) {
-  for (Token *t = tok; t->kind != TK_EOF; t = t->next) {
-    if (is_keyword(t))
-      t->kind = TK_KEYWORD;
-    else if (t->kind == TK_PP_NUM)
-      convert_pp_number(t);
-  }
-}
-
-// Initialize line info for all tokens.
-static void add_line_numbers(Token *tok) {
-  char *p = current_file->contents;
-  int n = 1;
-
-  do {
-    if (p == tok->loc) {
-      tok->line_no = n;
-      tok = tok->next;
-    }
-    if (*p == '\n')
-      n++;
-  } while (*p++);
-}
-
-Token *tokenize_string_literal(Token *tok, Type *basety) {
-  Token *t;
-  if (basety->size == 2)
-    t = read_utf16_string_literal(tok->loc, tok->loc);
-  else
-    t = read_utf32_string_literal(tok->loc, tok->loc, basety);
-  t->next = tok->next;
-  return t;
-}
-
-// Tokenize a given string and returns new tokens.
-Token *tokenize(File *file) {
-  current_file = file;
-
-  char *p = file->contents;
-  Token head = {0};
-  Token *cur = &head;
-
-  at_bol = true;
-  has_space = false;
-
-  while (*p) {
-    // Skip line comments.
-    if (startswith(p, "//")) {
-      p += 2;
-      while (*p != '\n')
-        p++;
-      has_space = true;
-      continue;
-    }
-
-    // Skip block comments.
-    if (startswith(p, "/*")) {
-      char *q = strstr(p + 2, "*/");
-      if (!q)
-        error_at(p, "unclosed block comment");
-      p = q + 2;
-      has_space = true;
-      continue;
-    }
-
-    // Skip newline.
-    if (*p == '\n') {
-      p++;
-      at_bol = true;
-      has_space = false;
-      continue;
-    }
-
-    // Skip whitespace characters.
-    if (isspace(*p)) {
-      p++;
-      has_space = true;
-      continue;
-    }
-
-    // Numeric literal
-    if (isdigit(*p) || (*p == '.' && isdigit(p[1]))) {
-      char *q = p++;
-      for (;;) {
-        if (p[0] && p[1] && strchr("eEpP", p[0]) && strchr("+-", p[1]))
-          p += 2;
-        else if (isalnum(*p) || *p == '.')
-          p++;
-        else
-          break;
-      }
-      cur = cur->next = new_token(TK_PP_NUM, q, p);
-      continue;
-    }
-
-    // String literal
-    if (*p == '"') {
-      cur = cur->next = read_string_literal(p, p);
-      p += cur->len;
-      continue;
-    }
-
-    // UTF-8 string literal
-    if (startswith(p, "u8\"")) {
-      cur = cur->next = read_string_literal(p, p + 2);
-      p += cur->len;
-      continue;
-    }
-
-    // UTF-16 string literal
-    if (startswith(p, "u\"")) {
-      cur = cur->next = read_utf16_string_literal(p, p + 1);
-      p += cur->len;
-      continue;
-    }
-
-    // Wide string literal
-    if (startswith(p, "L\"")) {
-      cur = cur->next = read_utf32_string_literal(p, p + 1, ty_int);
-      p += cur->len;
-      continue;
-    }
-
-    // UTF-32 string literal
-    if (startswith(p, "U\"")) {
-      cur = cur->next = read_utf32_string_literal(p, p + 1, ty_uint);
-      p += cur->len;
-      continue;
-    }
-
-    // Character literal
-    if (*p == '\'') {
-      cur = cur->next = read_char_literal(p, p, ty_int);
-      cur->val = (char)cur->val;
-      p += cur->len;
-      continue;
-    }
-
-    // UTF-16 character literal
-    if (startswith(p, "u'")) {
-      cur = cur->next = read_char_literal(p, p + 1, ty_ushort);
-      cur->val &= 0xffff;
-      p += cur->len;
-      continue;
-    }
-
-    // Wide character literal
-    if (startswith(p, "L'")) {
-      cur = cur->next = read_char_literal(p, p + 1, ty_int);
-      p += cur->len;
-      continue;
-    }
-
-    // UTF-32 character literal
-    if (startswith(p, "U'")) {
-      cur = cur->next = read_char_literal(p, p + 1, ty_uint);
-      p += cur->len;
-      continue;
-    }
-
-    // Identifier or keyword
-    int ident_len = read_ident(p);
-    if (ident_len) {
-      cur = cur->next = new_token(TK_IDENT, p, p + ident_len);
-      p += cur->len;
-      continue;
-    }
-
-    // Punctuators
-    int punct_len = read_punct(p);
-    if (punct_len) {
-      cur = cur->next = new_token(TK_PUNCT, p, p + punct_len);
-      p += cur->len;
-      continue;
-    }
-
-    error_at(p, "invalid token");
-  }
-
-  cur = cur->next = new_token(TK_EOF, p, p);
-  add_line_numbers(head.next);
-  return head.next;
-}
-
-// Returns the contents of a given file.
-/* this function is a bit wonkily implemented, and relies on a non-windows
- * thing, so we have our own in src/cmeta.c.
-static char *read_file(char *path) {
-  FILE *fp;
-
-  if (strcmp(path, "-") == 0) {
-    // By convention, read from stdin if a given filename is "-".
-    fp = stdin;
-  } else {
-    fp = fopen(path, "r");
-    if (!fp)
-      return NULL;
-  }
-
-  char *buf;
-  size_t buflen;
-  FILE *out = open_memstream(&buf, &buflen);
-
-  // Read the entire file.
-  for (;;) {
-    char buf2[4096];
-    int n = fread(buf2, 1, sizeof(buf2), fp);
-    if (n == 0)
-      break;
-    fwrite(buf2, 1, n, out);
-  }
-
-  if (fp != stdin)
-    fclose(fp);
-
-  // Make sure that the last line is properly terminated with '\n'.
-  fflush(out);
-  if (buflen == 0 || buf[buflen - 1] != '\n')
-    fputc('\n', out);
-  fputc('\0', out);
-  fclose(out);
-  return buf;
-}
-*/
-
-File **get_input_files(void) {
-  return input_files;
-}
-
-File *new_file(char *name, int file_no, char *contents) {
-  File *file = calloc(1, sizeof(File));
-  file->name = name;
-  file->display_name = name;
-  file->file_no = file_no;
-  file->contents = contents;
-  return file;
-}
-
-// Removes backslashes followed by a newline.
-static void remove_backslash_newline(char *p) {
-  int i = 0, j = 0;
-
-  // We want to keep the number of newline characters so that
-  // the logical line number matches the physical one.
-  // This counter maintain the number of newlines we have removed.
-  int n = 0;
-
-  while (p[i]) {
-    if (p[i] == '\\' && p[i + 1] == '\n') {
-      i += 2;
-      n++;
-    } else if (p[i] == '\n') {
-      p[j++] = p[i++];
-      for (; n > 0; n--)
-        p[j++] = '\n';
-    } else {
-      p[j++] = p[i++];
-    }
-  }
-
-  for (; n > 0; n--)
-    p[j++] = '\n';
-  p[j] = '\0';
-}
-
-static uint32_t read_universal_char(char *p, int len) {
-  uint32_t c = 0;
-  for (int i = 0; i < len; i++) {
-    if (!isxdigit(p[i]))
-      return 0;
-    c = (c << 4) | from_hex(p[i]);
-  }
-  return c;
-}
-
-// Replace \u or \U escape sequences with corresponding UTF-8 bytes.
-static void convert_universal_chars(char *p) {
-  char *q = p;
-
-  while (*p) {
-    if (startswith(p, "\\u")) {
-      uint32_t c = read_universal_char(p + 2, 4);
-      if (c) {
-        p += 6;
-        q += encode_utf8(q, c);
-      } else {
-        *q++ = *p++;
-      }
-    } else if (startswith(p, "\\U")) {
-      uint32_t c = read_universal_char(p + 2, 8);
-      if (c) {
-        p += 10;
-        q += encode_utf8(q, c);
-      } else {
-        *q++ = *p++;
-      }
-    } else if (p[0] == '\\') {
-      *q++ = *p++;
-      *q++ = *p++;
-    } else {
-      *q++ = *p++;
-    }
-  }
-
-  *q = '\0';
-}
-
-// NOTE modified API from upstream
-Token *tokenize_buf(const char *name, char *p) {
-  remove_backslash_newline(p);
-  convert_universal_chars(p);
-
-  // Save the filename for assembler .file directive.
-  static int file_no;
-  File *file = new_file((char *)name, file_no + 1, p);
-
-  // Save the filename for assembler .file directive.
-  input_files = realloc(input_files, sizeof(char *) * (file_no + 2));
-  input_files[file_no] = file;
-  input_files[file_no + 1] = NULL;
-  file_no++;
-
-  return tokenize(file);
-}