Add arena allocator to lexer

Author: timkaechele

src/herb.c

@@ -1,8 +1,10 @@
 #include "include/herb.h"
 #include "include/io.h"
 #include "include/lexer.h"
+#include "include/macros.h"
 #include "include/parser.h"
 #include "include/token.h"
+#include "include/util/hb_arena.h"
 #include "include/util/hb_array.h"
 #include "include/util/hb_buffer.h"
 #include "include/version.h"
@@ -11,8 +13,11 @@
 #include <stdlib.h>
 
 hb_array_T* herb_lex(const char* source) {
+  hb_arena_T allocator;
+  hb_arena_init(&allocator, KB(32));
+
   lexer_T lexer = { 0 };
-  lexer_init(&lexer, source);
+  lexer_init(&lexer, &allocator, source);
 
   token_T* token = NULL;
   hb_array_T* tokens = hb_array_init(128);
@@ -29,8 +34,11 @@ hb_array_T* herb_lex(const char* source) {
 AST_DOCUMENT_NODE_T* herb_parse(const char* source, parser_options_T* options) {
   if (!source) { source = ""; }
 
+  hb_arena_T allocator;
+  hb_arena_init(&allocator, MB(512));
+
   lexer_T lexer = { 0 };
-  lexer_init(&lexer, source);
+  lexer_init(&lexer, &allocator, source);
   parser_T parser = { 0 };
 
   parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;

src/include/lexer.h

@@ -3,8 +3,9 @@
 
 #include "lexer_struct.h"
 #include "token_struct.h"
+#include "util/hb_arena.h"
 
-void lexer_init(lexer_T* lexer, const char* source);
+void lexer_init(lexer_T* lexer, hb_arena_T* allocator, const char* source);
 token_T* lexer_next_token(lexer_T* lexer);
 token_T* lexer_error(lexer_T* lexer, const char* message);
 

src/include/lexer_struct.h

@@ -1,6 +1,7 @@
 #ifndef HERB_LEXER_STRUCT_H
 #define HERB_LEXER_STRUCT_H
 
+#include "util/hb_arena.h"
 #include "util/hb_string.h"
 
 #include <stdbool.h>
@@ -14,6 +15,7 @@ typedef enum {
 } lexer_state_T;
 
 typedef struct LEXER_STRUCT {
+  hb_arena_T* allocator;
   hb_string_T source;
 
   uint32_t current_line;

src/lexer.c

@@ -2,6 +2,7 @@
 #include "include/token.h"
 #include "include/utf8.h"
 #include "include/util.h"
+#include "include/util/hb_arena.h"
 #include "include/util/hb_buffer.h"
 #include "include/util/hb_string.h"
 
@@ -31,7 +32,9 @@ static bool lexer_stalled(lexer_T* lexer) {
   return lexer->stalled;
 }
 
-void lexer_init(lexer_T* lexer, const char* source) {
+void lexer_init(lexer_T* lexer, hb_arena_T* allocator, const char* source) {
+  lexer->allocator = allocator;
+
   if (source != NULL) {
     lexer->source = hb_string(source);
   } else {