6 #include "./tokenizer.h"
8 static struct Token token(const char *begin, const char *end)
10 struct Token token = {
18 static bool is_symbol_char(char x)
20 static const char forbidden_symbol_chars[] = {
21 '(', ')', '"', '\'', ';', '.'
23 static const size_t n = sizeof(forbidden_symbol_chars) / sizeof(char);
25 for (size_t i = 0; i < n; ++i) {
26 if (x == forbidden_symbol_chars[i] || isspace(x)) {
34 static const char *skip_whitespace(const char *str)
38 while(*str != 0 && isspace(*str)) {
45 static const char *next_quote(const char *str)
49 while(*str != 0 && *str != '"') {
56 static const char *skip_until_newline(const char *str)
60 while(*str != 0 && *str != '\n') {
67 static const char *next_non_symbol(const char *str)
71 while(*str != 0 && is_symbol_char(*str)) {
78 struct Token next_token(const char *str)
82 str = skip_whitespace(str);
84 return token(str, str);
87 while (*str != 0 && *str == ';') {
88 str = skip_until_newline(str + 1);
89 str = skip_whitespace(str);
96 return token(str, str + 1);
99 const char *str_end = next_quote(str + 1);
100 return token(str, *str_end == 0 ? str_end : str_end + 1);
104 return token(str, next_non_symbol(str + 1));