#include "reader.h" #include typedef enum { TOKEN_ZERO, TOKEN_LITERAL, TOKEN_PIPE, TOKEN_OR, TOKEN_AND, TOKEN_LINE_COMMENT, TOKEN_O_BRACKET, TOKEN_C_BRACKET, TOKEN_O_BRACES, TOKEN_C_BRACES, TOKEN_O_PARENTHESIS, TOKEN_C_PARENTHESIS, TOKEN_MULTIPLICATION, TOKEN_PLUS, TOKEN_MINUS, TOKEN_DIVISION } TokenType; typedef struct { TokenType type; String value; } Token; typedef struct { Arena *arena; u32 size; Token *list; } TokenList; TokenList token_list_new(Arena *arena, u64 capacity) { TokenList list; list.arena = arena; list.size = 0; list.list = arena_array(arena, Token, capacity); return list; } int token_compare_symbol(String *slice, Token *token) { if (string_compare_literal(slice, "|>") == 0) { token->type = TOKEN_PIPE; memcpy(&(token->value), slice, 2); return 1; } else if (string_compare_literal(slice, "||") == 0) { token->type = TOKEN_OR; memcpy(&(token->value), slice, 2); return 1; } else if (string_compare_literal(slice, "&&") == 0) { token->type = TOKEN_AND; memcpy(&(token->value), slice, 2); return 1; } else if (string_compare_literal(slice, "//") == 0) { token->type = TOKEN_LINE_COMMENT; memcpy(&(token->value), slice, 2); return 1; } if (string_compare_literal(slice, "}") == 0) { token->type = TOKEN_O_BRACES; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "{") == 0) { token->type = TOKEN_C_BRACES; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "[") == 0) { token->type = TOKEN_O_BRACKET; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "]") == 0) { token->type = TOKEN_C_BRACKET; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "(") == 0) { token->type = TOKEN_O_PARENTHESIS; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, ")") == 0) { token->type = TOKEN_C_PARENTHESIS; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "*") == 0) { token->type = TOKEN_MULTIPLICATION; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "+") == 0) { token->type = TOKEN_PLUS; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "-") == 0) { token->type = TOKEN_MINUS; memcpy(&(token->value), slice, 1); return 1; } else if (string_compare_literal(slice, "/") == 0) { token->type = TOKEN_DIVISION; memcpy(&(token->value), slice, 1); return 1; } return 0; } int tokenize(String slice, Token *token) { if (slice.size < 3 && token_compare_symbol(&slice, token) == 1) { return 1; } return 0; } TokenList tokennize_list(Arena *arena, Reader *reader) { u32 start = 0, end = 1, count = 0; String slice; Token token; TokenList list; memset(&token, 0, sizeof(token)); /* Count tokens */ do { slice.size = end - start; slice.buf = &(reader->buffer[start]); if (tokenize(slice, &token)) { count++; end = start; } end++; } while (reader_advance(reader, &start, &end)); reader_reset(reader); list = token_list_new(arena, count); count = 0; /* Process tokens */ do { slice.size = end - start; slice.buf = &(reader->buffer[start]); if (tokenize(slice, &token)) { memcpy(&(list.list[count++]), &token, sizeof(token)); } end++; } while (reader_advance(reader, &start, &end)); return list; }