diff --git a/CMakeLists.txt b/CMakeLists.txt index 9deee3a..7753c98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ add_subdirectory(libs/arena_allocator) add_subdirectory(libs/data) add_subdirectory(libs/progressbar) +add_subdirectory(libs/json) add_subdirectory(libs/mover) add_subdirectory(examples) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a4d2c20 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +compile_commands: + cmake -G Ninja -B build + +build: compile_commands + cmake --build build + +test: build +ifdef T + cd build && ctest --output-on-failure -R $(T) +else + cd build && ctest --output-on-failure +endif + +.PHONY: compile_commands build test diff --git a/docs/libs/containers/string.md b/docs/libs/containers/string.md index a0e9bc8..7e480bf 100644 --- a/docs/libs/containers/string.md +++ b/docs/libs/containers/string.md @@ -63,6 +63,12 @@ char *raw = string_to_primitive_underlying(s); printf("Raw: %s\n", raw); ``` +Note: +Since inside, string is just an `array_list`, it is not null terminated. Caller must use `string_length` along with `string_to_primitive_underlying` for this to work correctly. Or else use `string_to_primitive` which incurs copy and allocation cost. + +For comparisons, use `string_cmp_cstr` instead. + + ## API Reference | Function / Macro | Description | diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9d066f9..cf278aa 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,6 +1,2 @@ add_executable(progressbar progressbar.c) target_link_libraries(progressbar PRIVATE cisastor::progressbar m) -add_executable(array_list array_list.c) -target_link_libraries(array_list PRIVATE cisastor::utils) -target_link_libraries(array_list PRIVATE cisastor::data::containers) -target_link_libraries(array_list PRIVATE cisastor::arena) diff --git a/examples/array_list.c b/examples/array_list.c deleted file mode 100644 index b1ec81e..0000000 --- a/examples/array_list.c +++ /dev/null @@ -1,86 +0,0 @@ -#include "containers/array_list.h" -#include "containers/iterator.h" -#include "macros.h" - -#include -#include -#include - -struct foo { - int num; -}; - -typedef struct foo foo; - -array_list_init(foo); - -void foo_add_one(const void *const current_element, const void *result) { - const struct foo *const elem = current_element; - struct foo *elem_copy = map_copy(*elem); - elem_copy->num++; - send_to(*elem_copy, result); -} - -uint8_t foo_is_even(const void *const current_element) { - const struct foo *const elem = current_element; - if (elem->num % 2 == 0) { - return 1; - } - return 0; -} - -int main() { - struct array_list_int *al = array_list_create_int(); - assert(null != al); - array_list_append_int(al, 1); - array_list_append_int(al, 2); - array_list_append_int(al, 3); - assert(3 == *array_list_get_int_at(al, 2)); - - struct array_list_foo *al2 = array_list_create_foo(); - assert(null != al2); - struct foo f = {.num = 1}; - array_list_append_foo(al2, f); - f.num = 2; - array_list_append_foo(al2, f); - printf("at 1 %d\n", array_list_get_foo_at(al2, 1)->num); - f.num = 3; - array_list_append_foo(al2, f); - - for (struct iterator *it = iterator_begin(al2); - iterator_element(it) != iterator_end(it); iterator_increment(it)) { - printf("before mutation: %d\n", iterator_element_foo(it)->num); - } - - struct foo result = {}; - - for (struct iterator_mutating *itm = - iterator_map(iterator_begin(al2), foo_add_one, &result); - null != itm && - iterator_mutating_end(itm) != iterator_mutating_element(itm); - iterator_mutating_increment(itm)) { - printf("with foo_add_one: %d\n", - (cast(struct foo *, iterator_mutating_element(itm)))->num); - } - - for (struct iterator_mutating *itm = - iterator_filter(iterator_begin(al2), foo_is_even); - null != itm && - iterator_mutating_end(itm) != iterator_mutating_element(itm); - iterator_mutating_increment(itm)) { - printf("foo is even: %d\n", - (cast(struct foo *, iterator_mutating_element(itm)))->num); - } - - struct itertator *it = iterator_begin(al2); - for (struct iterator_mutating *itm = iterator_mutating_filter( - iterator_mutating_map(iterator_map(it, foo_add_one, &result), - foo_add_one, &result), - foo_is_even); - null != itm && - iterator_mutating_end(itm) != iterator_mutating_element(itm); - iterator_mutating_increment(itm)) { - printf("foo is even after plus 2: %d\n", - (cast(struct foo *, iterator_mutating_element(itm)))->num); - } -} diff --git a/libs/arena_allocator/arena.c b/libs/arena_allocator/arena.c index 83618cc..f393058 100644 --- a/libs/arena_allocator/arena.c +++ b/libs/arena_allocator/arena.c @@ -47,6 +47,10 @@ void *arena_alloc(size_t size) { // sizeof(char) == 1 byte most of the time; size_t required_size = size < ARENA_SIZ ? ARENA_SIZ : size * 2; arena.chunk = malloc(required_size); + if (arena.chunk == null) { + fprintf(stderr, "Failed to initialize Arena\n"); + return null; + } arena.size = required_size; arena.ptr = arena.chunk; atexit(arena_free); @@ -62,11 +66,16 @@ void *arena_alloc(size_t size) { current = new_arena; size_t new_size = size > ARENA_SIZ ? size * 2 : ARENA_SIZ; current->chunk = malloc(new_size); + if (current->chunk == null) { + perror("new_arena_chunk_allocation"); + return null; + } current->ptr = current->chunk; current->size = new_size; current->next_arena = null; } if (null == current->chunk || null == current->ptr) { + perror("no_arena_region"); return null; } void *allocated = current->ptr; diff --git a/libs/data/CMakeLists.txt b/libs/data/CMakeLists.txt index 06192bd..753ec14 100644 --- a/libs/data/CMakeLists.txt +++ b/libs/data/CMakeLists.txt @@ -6,6 +6,7 @@ add_library(cisastor_data_containers STATIC containers/iterator.c containers/string.c containers/string_builder.c + containers/stack.c ) target_include_directories(cisastor_data_containers INTERFACE @@ -55,3 +56,11 @@ cisastor_add_test( IMPLS containers/string_builder.c containers/string.c containers/array_list.c containers/iterator.c LIBS cisastor::arena cisastor::utils ) + +cisastor_add_test( + NAME stack_test + SRCS containers/stack_test.c + IMPLS containers/stack.c containers/array_list.c + LIBS cisastor::arena cisastor::utils +) + diff --git a/libs/data/containers/iterator.h b/libs/data/containers/iterator.h index 3aea70b..b8dd7b9 100644 --- a/libs/data/containers/iterator.h +++ b/libs/data/containers/iterator.h @@ -121,3 +121,6 @@ void *iterator_mutating_end(struct iterator_mutating *itm); #define for_each(it, list) \ for (struct iterator *it = iterator_begin(list); \ iterator_end(it) != iterator_element(it); iterator_increment(it)) + +#define for_each_it(it) \ + for (; iterator_end(it) != iterator_element(it); iterator_increment(it)) diff --git a/libs/data/containers/stack.c b/libs/data/containers/stack.c new file mode 100644 index 0000000..daac861 --- /dev/null +++ b/libs/data/containers/stack.c @@ -0,0 +1,28 @@ +#include "stack.h" +#include "array_list.h" + +stack *stack_create(size_t size) { return array_list_create(size); } + +byte *stack_peek(stack *s) { + struct array_list *al = cast(struct array_list *, s); + void *element = array_list_get_element_at(al, al->length - 1); + if (element == null) { + return null; + } + return element; +} + +byte *stack_pop(stack *s) { + void *element = stack_peek(s); + if (element == null) { + return null; + } + // array_list does not own anything, so this is fine, we aren't leaking memory + // by moving the cursor; + cast(struct array_list *, s)->length--; + return element; +} + +void stack_push(stack *s, byte *data) { array_list_append(s, data); } + +size_t stack_size(stack *s) { return array_list_get_length(s); } diff --git a/libs/data/containers/stack.h b/libs/data/containers/stack.h new file mode 100644 index 0000000..f0501ee --- /dev/null +++ b/libs/data/containers/stack.h @@ -0,0 +1,35 @@ +#pragma once + +#include "array_list.h" +#include "macros.h" + +typedef struct array_list stack; + +stack *stack_create(size_t size); + +byte *stack_pop(stack *); +byte *stack_peek(stack *); +void stack_push(stack *, byte *data); + +size_t stack_size(stack *); + +#define stack_init(type) \ + typedef stack stack_##type; \ + static inline stack_##type *stack_create_##type() { \ + return stack_create(sizeof(type)); \ + } \ + static inline void stack_push_##type(stack_##type *s, type data) { \ + stack_push(s, cast(byte *, &data)); \ + } \ + static inline void stack_push_##type##_ptr(stack_##type *s, type *data) { \ + stack_push(s, cast(byte *, data)); \ + } \ + static inline type *stack_pop_##type(stack_##type *s) { \ + return cast(type *, stack_pop(s)); \ + } \ + static inline type *stack_peek_##type(stack_##type *s) { \ + return cast(type *, stack_peek(s)); \ + } + +stack_init(int); +stack_init(char); diff --git a/libs/data/containers/stack_test.c b/libs/data/containers/stack_test.c new file mode 100644 index 0000000..40506ba --- /dev/null +++ b/libs/data/containers/stack_test.c @@ -0,0 +1,41 @@ +#include "testing.h" + +#include "stack.h" + +IT(should_enstack_elements_and_pop) { + stack_int *q = stack_create_int(); + for (int i = 1; i <= 10000; i++) { + stack_push_int(q, i); + } + cassert(stack_size(q) == 10000, "invalid stack size, should be 10000"); + for (int i = 10000; i > 0; i--) { + int *_got = stack_pop_int(q); + cassertf(_got != null, "popped null data at index %d\n", i); + int got = *_got; + cassertf(got == i, "expected %d got %d\n", i, got); + } + cassert(stack_size(q) == 0, "size should be 0 now"); +} + +IT(should_peek_pop_correctly) { + stack_int *s = stack_create_int(); + stack_push_int(s, 0); + int *i = null; + i = stack_peek_int(s); + cassert(i != null, "peek should not be null, value should be 0 at the " + "beginning that is peeked into"); + cassert(*i == 0, "should peek value 0"); + stack_push_int(s, 1); + i = stack_peek_int(s); + cassert(i != null, + "peek should not be null, value should be 1 that is peeked into"); + cassert(*i == 1, "peek should be 1"); + i = stack_pop_int(s); + cassert(i != null, + "pop should not be null, value should be 1 that is popped"); + cassert(*i == 1, "pop should be 1"); + i = stack_peek_int(s); + cassert(i != null, + "peek should not be null, value should be 0 that is peeked into"); + cassert(*i == 0, "peek should be 0"); +} diff --git a/libs/data/containers/string.c b/libs/data/containers/string.c index a5bbd56..ba8d414 100644 --- a/libs/data/containers/string.c +++ b/libs/data/containers/string.c @@ -71,5 +71,15 @@ char *string_to_primitive(string *str) { } char *const string_to_primitive_underlying(string *str) { + // Since inside, string is just an array_list, it is not null terminated. + // Caller must use string_length along with _to_primitive_underlying for this + // to work correctly. Or else use string_to_primitive which incurs copy and + // allocation cost. + // For comparisons, use string_cmp_cstr instead. return cast(struct array_list *, str)->region; } + +// str[n]cmp for using string with cstrs +uint8_t string_cmp_cstr(string *str, const char *const cstr) { + return strncmp(string_to_primitive_underlying(str), cstr, string_length(str)); +} diff --git a/libs/data/containers/string.h b/libs/data/containers/string.h index 677f860..f893300 100644 --- a/libs/data/containers/string.h +++ b/libs/data/containers/string.h @@ -25,3 +25,5 @@ string *string_from_array_list(struct array_list_char *al); char *string_to_primitive(string *str); char *const string_to_primitive_underlying(string *str); + +uint8_t string_cmp_cstr(string *str, const char *const cstr); diff --git a/libs/data/containers/string_builder.c b/libs/data/containers/string_builder.c index f37a853..e4ddfe0 100644 --- a/libs/data/containers/string_builder.c +++ b/libs/data/containers/string_builder.c @@ -2,6 +2,7 @@ #include "array_list.h" #include "string.h" +#include #include string_builder *string_builder_create() { @@ -22,6 +23,10 @@ string *string_builder_build(string_builder *sb) { capacity += strlen(*array_list_get_const_char_ptr_at(sb, i)); } string *str = cast(string *, array_list_create_char_with_capacity(capacity)); + if (str == null) { + perror("string_builder_build_new_string"); + return null; + } for (int i = 0; i < array_list_get_length(sb); i++) { const char *cstr = *array_list_get_const_char_ptr_at(sb, i); for (int j = 0; cstr[j] != '\0'; j++) { diff --git a/libs/data/containers/string_builder_test.c b/libs/data/containers/string_builder_test.c index 54c2fc8..72387d8 100644 --- a/libs/data/containers/string_builder_test.c +++ b/libs/data/containers/string_builder_test.c @@ -1,5 +1,5 @@ -#include "testing.h" #include "string_builder.h" +#include "testing.h" IT(should_create_string_builder) { string_builder *sb = string_builder_create(); @@ -10,10 +10,10 @@ IT(should_join_strings) { string_builder *sb = string_builder_create(); string_builder_join(sb, "hello "); string_builder_join(sb, "world"); - + string *str = string_builder_build(sb); cassert(string_length(str) == 11, "length should be 11"); - + string *expected = string_create("hello world"); cassert(string_cmp(str, expected) == 0, "string mismatch"); } @@ -23,10 +23,11 @@ IT(should_handle_many_joins) { for (int i = 0; i < 1000; i++) { string_builder_join(sb, "a"); } - + string *str = string_builder_build(sb); cassert(string_length(str) == 1000, "length should be 1000"); - for(int i = 0; i < 1000; i++) { - cassertf(string_char_at(str, i) == 'a', "mismatch at idx %d", i); + for (int i = 0; i < 1000; i++) { + cassertf(string_char_at(str, i) == 'a', "mismatch at idx %d", i); } } + diff --git a/libs/data/containers/string_test.c b/libs/data/containers/string_test.c index 0a14511..5e3e907 100644 --- a/libs/data/containers/string_test.c +++ b/libs/data/containers/string_test.c @@ -1,5 +1,5 @@ -#include "testing.h" #include "string.h" +#include "testing.h" #include IT(should_create_string) { @@ -25,8 +25,8 @@ IT(should_compare_strings) { cassert(string_cmp(s1, s2) == 0, "s1 should equal s2"); cassert(string_cmp(s1, s3) != 0, "s1 should not equal s3"); - // string_cmp implementation uses strncmp with min length, so "abc" vs "ab" might return 0? - // Let's test what it actually does. + // string_cmp implementation uses strncmp with min length, so "abc" vs "ab" + // might return 0? Let's test what it actually does. } IT(should_extract_substring) { @@ -44,8 +44,48 @@ IT(should_concatenate_strings) { string *res = string_concat(s1, s2); /* printf("%s\n", string_to_primitive(res)); */ - + cassert(string_length(res) == 11, "length mismatch after concat"); cassert(string_char_at(res, 0) == 'h', "expected 'h'"); cassert(string_char_at(res, 6) == 'w', "expected 'w'"); } + +IT(should_return_correct_string_length) { + string *s1 = string_create(""); + for (size_t i = 1; i <= 10000; i++) { + string_append_char(s1, i); + } + cassert(string_length(s1) == 10000, "length should be 10000"); +} + +IT(should_return_the_same_for_copy_and_no_copy_primitive_functions) { + string *s1 = string_create("hello world"); + // using strncmp, see comment under string_to_primitive_underlying + cassert(strncmp(string_to_primitive_underlying(s1), string_to_primitive(s1), + string_length(s1)) == 0, + "Should be equal to \"hello world\""); +} + +IT(should_compare_string_instances_correctly) { + string *s1 = string_create("hello world"); + string *s2 = string_create("hello world"); + cassert(string_cmp(s1, s2) == 0, "s1 and s2 should be equal"); +} + +/* + * The following test is expected to fail, see comment under _underlying +function. + +IT(should_be_null_terminated_for_underlying_primitive_function) { + string *s1 = string_create("hello world"); + size_t length = string_length(s1); + cassert(string_to_primitive_underlying(s1)[length] == '\0', + "to_primitive_underlying should end string with null character"); +} +*/ + +IT(should_compare_cstrings_correctly) { + string *s1 = string_create("hello world"); + cassert(string_cmp_cstr(s1, "hello world") == 0, + "s1 should be equal to \"hello world\""); +} diff --git a/libs/json/CMakeLists.txt b/libs/json/CMakeLists.txt new file mode 100644 index 0000000..7e6895e --- /dev/null +++ b/libs/json/CMakeLists.txt @@ -0,0 +1,45 @@ +cmake_minimum_required(VERSION 3.21) + +add_library(cisastor_json STATIC + lexer.c + parser.c +) + +add_executable(cisastor_json_generator generator/generator.c) +target_link_libraries(cisastor_json_generator PRIVATE cisastor::utils) +target_link_libraries(cisastor_json_generator PRIVATE cisastor::arena) +target_link_libraries(cisastor_json_generator PRIVATE cisastor::data::containers) + +target_link_libraries(cisastor_json PRIVATE cisastor::utils) +target_link_libraries(cisastor_json PRIVATE cisastor::arena) +target_link_libraries(cisastor_json PRIVATE cisastor::data::containers) + +target_include_directories(cisastor_json PUBLIC + $ + $ +) + +add_library(cisastor::json ALIAS cisastor_json) + +include(${PROJECT_SOURCE_DIR}/libs/testing/CisastorTesting.cmake) + +cisastor_add_test( + NAME lexer_test + SRCS lexer_test.c + IMPLS lexer.c + LIBS cisastor::arena cisastor::utils cisastor::data::containers +) + +cisastor_add_test( + NAME parser_test + SRCS parser_test.c + IMPLS parser.c lexer.c + LIBS cisastor::arena cisastor::utils cisastor::data::containers +) + +cisastor_add_test( + NAME grammar_test + SRCS grammar_test.c + IMPLS grammar.c lexer.c + LIBS cisastor::arena cisastor::utils cisastor::data::containers +) diff --git a/libs/json/context.h b/libs/json/context.h new file mode 100644 index 0000000..a20709a --- /dev/null +++ b/libs/json/context.h @@ -0,0 +1,16 @@ +#pragma once + +// While walking, parser sets up state, these are +// Expectations -> transformations +// E.g., expectation -> closing '"', transformation -> new token in the ast + +#include "containers/iterator.h" + +struct context { +}; + +struct transformer_queue { +}; + + +// rule diff --git a/libs/json/generator/generator.c b/libs/json/generator/generator.c new file mode 100644 index 0000000..9d5ef94 --- /dev/null +++ b/libs/json/generator/generator.c @@ -0,0 +1,263 @@ +/* vim: set tabstop=2 shiftwidth=2 expandtab: */ +#include +#include +#include +#include +#include +#include + +#include "macros.h" +#include "containers/string_view.h" +#include "arena.h" + +enum token_kind { + keyword, + identifier, + star, + left_brace, + right_brace, + semicolon, + cisastor_comment, +}; + +struct token { + enum token_kind kind; + void *data; +}; + +char *token_kind_to_string(enum token_kind kind) { + switch (kind) { + case keyword: + return "KEYWORD"; + case identifier: + return "IDENTIFIER"; + case star: + return "STAR"; + case left_brace: + return "LEFT_BRACE"; + case right_brace: + return "RIGHT_BRACE"; + case semicolon: + return "SEMICOLON"; + case cisastor_comment: + return "GENERATOR_START"; + default: + return null; + } +} + +void _print_token(struct token *tok) { + printf("[%s(%s)]", token_kind_to_string(tok->kind), cast(char *, tok->data)); +} + +struct token_node { + struct token *token; + struct token_node *next; +}; + +struct token_node *token_node_create(enum token_kind kind, void *data) { + struct token_node *node = arena_alloc(sizeof(struct token_node)); + if (null == node) { + return null; + } + + node->token = arena_alloc(sizeof(struct token)); + if (null == node->token) { + return null; + } + node->token->kind = kind; + node->token->data = data; + return node; +} + +enum tokenize_state { + state_expecting_magic_comment, + state_expecting_struct, + state_inside_struct, +}; + +void tokenizing_state_advance(enum tokenize_state *state) { + switch (*state) { + case state_expecting_magic_comment: + *state = state_expecting_struct; + break; + case state_expecting_struct: + *state = state_inside_struct; + break; + case state_inside_struct: + *state = state_expecting_magic_comment; + break; + } +} + +void emit_node_or_fail(struct token_node **root, enum token_kind kind, + void *data) { + struct token_node *node = token_node_create(kind, data); + if (null == node) { + perror("token_allocation_failed"); + exit(1); + } + (*root)->next = node; + *root = node; +} + +struct token_node *tokenize(char *stream) { + enum tokenize_state state = state_expecting_magic_comment; + + // FIXME: waste of a node + struct token_node *root = token_node_create(cisastor_comment, ""); + if (null == root) { + return null; + } + + struct token_node *current_node = root; + + struct string_view *sv = string_view_create(stream); + + int curr = 0; + while ((curr = string_view_next_char(sv)) != '\0') { + string_view_consume_chars( + sv, " \t"); // skip until an alphanumeric char appears; + switch (state) { + case state_expecting_magic_comment: + if (curr != '/') { + continue; + } + int pos = string_view_current_pos(sv); + if (pos == 0 || string_view_char_at(sv, pos - 1) == 10) { + // newline or first line, comment is valid + int next = string_view_next_char(sv); + if (next == '/') { + const char *const look_for = "cisastor::json generate"; + + int matched = string_view_compare_word(sv, look_for); + + if (matched == 0) { + // read the rest + emit_node_or_fail(¤t_node, cisastor_comment, "generate"); + tokenizing_state_advance(&state); + } + + string_consume_till_end_of_line(sv); + continue; + } + } + break; + case state_expecting_struct: + if (curr != 's') { + continue; + } + if (string_view_compare_word(sv, "truct") != 0) { + fprintf(stderr, "tokenizer error: expected keyword 'struct' got '%s'\n", + string_view_consume_word(sv)); + return null; + } + emit_node_or_fail(¤t_node, keyword, "struct"); + + string_view_consume_chars(sv, " \t\n"); + + const char *const struct_name = string_view_consume_word(sv); + if (null == struct_name) { + return null; + } + + emit_node_or_fail(¤t_node, identifier, cast(void *, struct_name)); + + string_view_consume_chars(sv, " \t\n"); + + int expected_left_brace = string_view_next_char(sv); + if ('\0' == expected_left_brace) { + fprintf(stderr, + "tokenizer error: expected left opening brace, got EOF\n"); + return null; + } else if ('{' != expected_left_brace) { + fprintf(stderr, + "tokenizer error: expected left opening brace, got '%c'\n", + expected_left_brace); + return null; + } + + emit_node_or_fail(¤t_node, left_brace, "{"); + string_view_consume_chars(sv, " \t\n"); + tokenizing_state_advance(&state); + break; + case state_inside_struct: + // until an alphanumeric appears + string_view_consume_chars(sv, " \t\n"); + + if (curr == '}') { + emit_node_or_fail(¤t_node, right_brace, "}"); + cast(void, string_view_next_char(sv)); // consume before ending; + tokenizing_state_advance(&state); + break; + } + // w1 w2 w3 + // enum type iden; + // type iden; + const char *word = string_view_consume_word(sv); + string_view_consume_chars(sv, " \t\n"); + { + char *word_1 = string_create_empty(strlen(word) + 1); + word_1[0] = curr; + strncpy(word_1 + 1, word, strlen(word)); + emit_node_or_fail(¤t_node, keyword, cast(void *, word_1)); + } + const char *word2 = string_view_consume_word(sv); + string_view_consume_chars(sv, " \t\n"); + + char *identifier_str = null; + + if (string_view_peek_char(sv) == ';') { + // end of line; + identifier_str = word2; + } else { + emit_node_or_fail(¤t_node, keyword, cast(void *, word2)); + const char *word3 = string_view_consume_word(sv); + string_view_consume_chars(sv, " \t\n"); + identifier_str = word3; + } + string_view_consume_chars(sv, " \t\n"); + if (string_view_peek_char(sv) == '*') { + cast(void, string_view_next_char(sv)); + emit_node_or_fail(¤t_node, star, "*"); + } + string_view_consume_chars(sv, " \t\n"); + emit_node_or_fail(¤t_node, identifier, identifier_str); + int c = 0; + if ((c = string_view_next_char(sv)) != ';') { + fprintf(stderr, + "tokenizer error: expected statement terminator ';', got " + "'%c'\n", + c); + return null; + } + emit_node_or_fail(¤t_node, semicolon, ";"); + + break; + } + } + + return root; +} + +void tokenize_and_print(char *stream) { + struct token_node *root = tokenize(stream); + assert(root != null && "root should not be empty"); + struct token_node *node = + root->next; // skip the first one, wasted resource yes; + while (null != node) { + _print_token(node->token); + node = node->next; + } +} + +int main() { + tokenize_and_print("//cisastor::json generate\nstruct user {int num;}"); + putchar(10); + tokenize_and_print( + "//cisastor::json generate\n\n struct address\n{\nint num;}"); + putchar(10); + tokenize_and_print( + "//cisastor::json generate\n\n struct address\n{\nchar * road;}"); + return 0; +} diff --git a/libs/json/grammar.c b/libs/json/grammar.c new file mode 100644 index 0000000..1622f56 --- /dev/null +++ b/libs/json/grammar.c @@ -0,0 +1,177 @@ +#include "grammar.h" +#include "arena.h" +#include "macros.h" +#include "token.h" + +#include "containers/stack.h" +#include "containers/string.h" +#include "containers/string_builder.h" + +// for unreachable +#include + +const uint8_t token_types_length_plus_one = _token_types_count + 1; +const uint8_t state_types_length_plus_one = _state_types_count + 1; + +const uint8_t default_token_entry_index = _token_types_count; +const uint8_t default_state_entry_index = _state_types_count; // unused + +typedef enum { + state_pop = -1, + state_invalid = -2, +} _grammar_special_transition; + +struct _ge { + stack__parser_state *_context; +}; + +struct _ge *_validator_create() { + struct _ge *ge = aalloc(sizeof(struct _ge)); + if (ge == null) + return null; + stack__parser_state *context = stack_create__parser_state(); + if (context == null) + return null; + ge->_context = context; + stack_push__parser_state(context, begin); + return ge; +} + +void new_context(struct _ge *ge, _parser_state ctx) { + stack_push__parser_state(ge->_context, ctx); +} + +void end_context(struct _ge *ge) { stack_pop__parser_state(ge->_context); } + +string *invalid_token_error(struct token *token, const char *const append) { + return string_builder_build(string_builder_join( + string_builder_join( + string_builder_join( + string_builder_join(string_builder_create(), "Invalid token "), + token_to_string(token)), + ", "), + append)); +} + +_grammar_result *result_is_ok(_grammar_result *result, struct _ge *ge, + enum state state) { + result->ok = 1; + result->state = state; + new_context(ge, state); + return result; +} + +_grammar_result *result_is_ok_last_state(_grammar_result *result, + struct _ge *ge) { + result->ok = 1; + result->state = stack_size(ge->_context) == 0 + ? end + : *stack_pop__parser_state(ge->_context); + return result; +} + +_grammar_result *result_is_not_ok(_grammar_result *result, + string *diagnostics) { + result->ok = 0; + result->diagnostic = diagnostics; + return result; +} + +struct _grammar_rule; + +typedef _grammar_result *(*_grammar_rule_build_grammar_result_t)( + struct _grammar_rule *, struct _ge *, struct token *); + +struct _grammar_rule { + _grammar_rule_build_grammar_result_t _build; + const char *const _diagnostics; + _grammar_result _template; +}; + +uint8_t _grammar_rule_is_empty(struct _grammar_rule *rule) { + return rule->_build == null; +} + +_grammar_result *_grammar_rule_build_grammar_result(struct _grammar_rule *_rule, + struct _ge *_ge, + struct token *token) { + if (_rule->_template.ok) { + if (_rule->_template.state == state_pop) { + return result_is_ok_last_state(&_rule->_template, _ge); + } + return result_is_ok(&_rule->_template, _ge, _rule->_template.state); + } + + // FIXME uncomment once all tokens comint to this layer is filtered to only + // what it concerns itself with + // + // if (_rule->_template.state != state_invalid) { + // unreachable("for .ok == 0, state should be -2"); + // } + + return result_is_not_ok(&_rule->_template, + invalid_token_error(token, _rule->_diagnostics)); +} + +#define TO(is_ok, next, diag) \ + { \ + ._build = &_grammar_rule_build_grammar_result, ._diagnostics = diag, \ + ._template = { \ + .ok = is_ok, \ + .state = next, \ + } \ + } + +static struct _grammar_rule _grammar_context_table + [state_types_length_plus_one][token_types_length_plus_one] = { + [begin][left_brace] = TO(1, in_object, null), + [begin][left_square_bracket] = TO(1, in_list, null), + [begin][default_token_entry_index] = TO( + 0, state_invalid, "expected '{' or '[' as beginning of document"), + + // --- + + [in_object][quote] = TO(1, in_key, null), + [in_object][right_brace] = TO(1, state_pop, null), + [in_object][default_token_entry_index] = + TO(0, state_invalid, + "expected start of key '\"' or end of object '}'"), + + [in_key][quote] = TO(1, state_pop, null), + + [in_list][right_square_bracket] = TO(1, state_pop, null), +}; + +typedef enum token_type token_type; +array_list_init(token_type); + +struct _grammar_rule *_grammar_rule_get(_parser_state state, + enum token_type token_type) { + struct _grammar_rule *rule = &_grammar_context_table[state][token_type]; + if (_grammar_rule_is_empty(rule)) + return &_grammar_context_table[state][default_token_entry_index]; + return rule; +} + +struct _grammar_rule * +_grammar_rule_get_no_fallback(_parser_state state, enum token_type token_type) { + struct _grammar_rule *rule = &_grammar_context_table[state][token_type]; + if (_grammar_rule_is_empty(rule)) + return null; + return rule; +} + +_grammar_result *validate_token(struct _ge *ge, struct token *token) { + typedef stack__parser_state *context; + context ctx = ge->_context; + _parser_state current_state = *stack_peek__parser_state(ctx); + struct _grammar_rule *rule = _grammar_rule_get(current_state, token->type); + _grammar_result *result = rule->_build(rule, ge, token); // begin, in_object + for (current_state = *stack_peek__parser_state(ctx), + rule = _grammar_rule_get_no_fallback(current_state, token->type); + rule != null; current_state = *stack_peek__parser_state(ctx), + rule = _grammar_rule_get_no_fallback(current_state, token->type)) { + result = rule->_build(rule, ge, token); + } + return result; +} diff --git a/libs/json/grammar.h b/libs/json/grammar.h new file mode 100644 index 0000000..2d7fc5f --- /dev/null +++ b/libs/json/grammar.h @@ -0,0 +1,136 @@ +#include "containers/stack.h" +#include "containers/string.h" + +#include "token.h" + +// what is a grammar? +// It tells what is valid through context, context that is attached to the word +// befire and after. But from a token's perspective, just the previous and next +// token's context can validate a grammar incorrectly. For example, `":"` is +// valid, but if these three tokens are like `{"key"":"`, then grammar is +// invalid. So, essentially, current and next token's are not enough. We need +// another piece of context that is an accumulation of all the contexts before. + +// state of a grammar, think about accumulating multiple contexts. +// transition of this state is controlled and validated according to the +// language grammar +typedef enum state { + begin, + + in_object, + in_list, + + in_kv_pair, + + in_key, + in_kv_delimiter, + in_value, + in_kv_pair_delimiter, + + end, + + _state_types_count, +} _parser_state; + +// in_object, in_kv_pair, in_key +// in_object, in_kv_pair, in_kv_delimiter +// in_object, in_kv_pair, in_value +// in_object, in_kv_pair, in_kv_pair_delimiter +// ... +// [in_kv_pair_delimiter][comma] = TO(1, state_pop, null), +// [in_kv_pair_delimiter][left_brace] = TO(1, {state_pop, state_pop, end}, null} +// [in_kv_pair_delimiter][left_square_bracket] = TO(1, {state_pop, state_pop, +// end}, null} +// so, we need a recursive rule validation +// [in_kv_pair_delimiter][left_brace] = TO(1, state_pop, null}, // in_object, +// in_kv_pair +// [in_kv_pair][left_brace] = TO(1, state_pop, null}, // in_object +// [in_object][left_brace] = TO(1, state_pop, null), // {begin} // the shadow +// state +// [begin][left_brace] = TO(1, end, null), // {begin,end} grammar is done +// [end][default_token_entry_index] = TO(0, state_invalid /* same as token +// invalid */, "invalid token detected after json document ended"), + +typedef struct { + uint8_t ok; + union { + _parser_state state; + string *diagnostic; + }; +} _grammar_result; + +// the "accumulation" is approached through a stack. +// One context can be shadowed by another, as far as the current cursor for the +// tokens is considered. +// the stack itself describes a subset of the language grammar pretty well, for +// example {in_key, in_object} is not a valid state, given the state should +// always start with either in_object or in_list (for ours at least). In other +// words, our state stack willfully describes the spec of the grammar the parser +// should and does support. +stack_init(_parser_state); + +// I'd prefer to not leak the state stack, +// and, especially not have a global state stack when a parsing can happen +// multiple times for multiple objects; Not that thread safety is a major +// concern yet, nor this change will make parsing thrread safe. More importantly +// i don't know yet. For the time being, what I know is that global state for +// parsing == bad, local state == helpful for later thraed safety audit. +struct _ge; + +// for_each(it, tokens) { +// struct token* token = next_non_empty_token(it); +// string* diag = validate_token(token); +// if (diag != null) { +// fprintf(stderr, "Invalid token \"%s\": %s\n", string_to_primitive(diag)); +// return null; +// } +// // token validated, now we need to know what to do about this token +// } +// +// We could have an "action" which dicated what to do with the token, but that's +// hard to quantify for all types of grammars. Instead, we expose the states +// directly to caller as part of the result struct; +// +// _grammar_result * result = validate_token(token); +// if (!result->ok) { +// fprintf(stderr, "Invalid token \"%s\": %s\n", +// string_to_primitive(result->diagnostic)); return null; +// } +// +// At any point we need to validate the current token against the current +// accumulated global context, which should complete the "lookback" portion of +// the stream. +// What we must ensure is HOW we process the token. In other words seperating +// the "grammar" from what the "parser" will do. Grammar either allows or +// disallows, enough diagnostics to tell the user why disallowed. parser knows +// how to accumulate the tokens into the final struct. +_grammar_result *validate_token(struct _ge *, struct token *token); +// ^ returns a diagnostic message, null if is allowed + +/* + _parser_state current_state = *stack_pop__parser_state(ctx); + // we expect no whitespaces here + switch (current_state) { + case in_object: + // need a key, logical next state is in_key transition or end of the object + switch (token->type) { + case quote: + return result_is_ok(result, ge, in_key); + case right_brace: + end_context(ge); + printf("here\n"); + return result_is_ok_last_state(result, ge); + default: + return result_is_not_ok( + result, invalid_token_error(token, "expected start of key '\"'")); + * ---- + * Looking at the code above, it's a table, like a bool table. + * row = current_state + * column = token->type + * Instead of a imperative switches and ifs, a declarative way of defining and + using a table makes more sense here. + * Likely a container type. + * Yes, hashmap with a tuple key is essentially a table. + * No, simple 2d table + * +*/ diff --git a/libs/json/grammar_test.c b/libs/json/grammar_test.c new file mode 100644 index 0000000..7a4b82a --- /dev/null +++ b/libs/json/grammar_test.c @@ -0,0 +1,77 @@ +#include "testing.h" + +#include "grammar.h" +#include "token.h" + +struct _ge *_validator_create(); + +static struct token tok(enum token_type type) { + struct token tok = {.type = type}; + return tok; +} + +IT(should_validate_empty_objects) { + struct _ge *ge = _validator_create(); + + cassert(validate_token(ge, &cast(struct token, {.type = left_brace}))->ok, + "left brace should be valid document start"); + cassert(validate_token(ge, &cast(struct token, {.type = right_brace}))->ok, + "right brace should be valid document end"); +} + +IT(should_validate_empty_lists) { + struct _ge *ge = _validator_create(); + cassert(validate_token(ge, &cast(struct token, {.type = left_square_bracket})) + ->ok, + "left square bracket should be valid document start"); + cassert( + validate_token(ge, &cast(struct token, {.type = right_square_bracket})) + ->ok, + "right square bracket should be valid document end"); +} + +IT(should_validate_flat_json_object_with_all_typed_values) { + struct _ge *ge = _validator_create(); + struct token token = tok(left_brace); + _grammar_result *result = validate_token(ge, &token); + cassert(result->ok, "{ should be valid json start"); + cassert(result->state == in_object, "{ should start in_object context"); + token = tok(quote); + result = validate_token(ge, &token); + cassert(result->ok, "{\" should be valid json"); + cassert(result->state == in_key, "{ should start in_key context"); + token = tok(quote); + result = validate_token(ge, &token); + cassert(result->ok, "{\"\" should be valid json"); + cassert(result->state == delim, "{\"\" should set context to delim"); + token = tok(colon); + result = validate_token(ge, &token); + cassert(result->ok, "{\"\": should be valid json"); + cassert(result->state == in_value, "{\"\": should set context to in_value"); + token = tok(number); + result = validate_token(ge, &token); + cassert(result->ok, "{\"\":8 should be valid json"); + cassert(result->state == delim, "{\"\":8 should set context to delim"); + token = tok(comma); + result = validate_token(ge, &token); + cassert(result->ok, "{\"\":8, should be valid json"); + cassert(result->state == in_object, + "{\"\":8, should set context to in_object"); + token = tok(quote); + validate_token(ge, &token); + token = tok(quote); + validate_token(ge, &token); + token = tok(colon); + validate_token(ge, &token); + // {\"\":8,\"\": <- so far + token = tok(quote); + result = validate_token(ge, &token); + cassert(result->ok, "{\"\":8,\"\":\" should be valid json"); + cassert(result->state == in_value, + "{\"\":8,\"\":\" should set context to in_value"); + token = tok(quote); + result = validate_token(ge, &token); + cassert(result->ok, "{\"\":8,\"\":\"\" should be valid json"); + cassert(result->state == in_object, + "{\"\":8,\"\":\"\" should set context to in_object"); +} diff --git a/libs/json/json.h b/libs/json/json.h new file mode 100644 index 0000000..3cf951f --- /dev/null +++ b/libs/json/json.h @@ -0,0 +1,41 @@ +#include "containers/array_list.h" +#include "containers/string.h" + +#include + +enum json_kind { + json_null, + json_number, + json_bool, + json_string, + json_list, + json_object, +}; + +struct json_list { + // json_value + struct array_list *items; +}; + +struct json_object { + // json_field + struct array_list *fields; +}; + +struct json_value { + enum json_kind kind; + union { + uint8_t boolean; + double number; + string *string; + struct json_list *list; + struct json_object *object; + }; +}; + +struct json_field { + string *key; + struct json_value *value; +}; + +struct json_value *parse_string(struct iterator *it); diff --git a/libs/json/lexer.c b/libs/json/lexer.c new file mode 100644 index 0000000..caacfc7 --- /dev/null +++ b/libs/json/lexer.c @@ -0,0 +1,198 @@ +#include "token.h" + +#include "arena.h" +#include "containers/string_view.h" +#include "macros.h" + +#include +#include + +char *token_type_to_string(enum token_type type) { + switch (type) { + case left_brace: + return "LEFT_BRACE"; + case quote: + return "QUOTE"; + case colon: + return "COLON"; + case words: + return "WORDS"; + case number: + return "NUMBER"; + case boolean: + return "BOOLEAN"; + case nullish: + return "NULL"; + case comma: + return "COMMA"; + case right_brace: + return "RIGHT_BRACE"; + case newline: + return "NEWLINE"; + case space: + return "SPACE"; + case left_square_bracket: + return "LEFT_SQUARE_BRACKET"; + case right_square_bracket: + return "RIGHT_SQUARE_BRACKET"; + default: + return "UNKNOWN"; + } +} + +char *token_to_string(struct token *tok) { + char *tok_str = token_type_to_string(tok->type); + const char *const extra_chars = "[]()"; + size_t length = strlen(tok_str) + strlen(extra_chars); + switch (tok->type) { + case words: + if (null == tok->value) { + tok->value = "unknown"; + } + length += strlen(tok->value); + char *word = string_create_empty(length); + if (null == word) { + return null; + } + sprintf(word, "[%s(%s)]", tok_str, tok->value); + return word; + case number: { + int digits = 1; + int num = tok->num; + while (num /= 10) + digits++; + char *word = string_create_empty(length + digits); + if (null == word) { + return null; + } + sprintf(word, "[%s(%d)]", tok_str, tok->num); + return word; + } + case boolean: { + char *value = tok->num ? "true" : "false"; + char *word = string_create_empty(length + strlen(value)); + if (null == word) { + return null; + } + sprintf(word, "[%s(%s)]", tok_str, value); + return word; + } + default: { + char *word = string_create_empty(strlen(tok_str) + 2); + if (null == word) { + return null; + } + sprintf(word, "[%s]", tok_str); + return word; + } + } +} + +void append_token_to_token_list_or_fail(struct array_list_token *token_list, + enum token_type type, char *value, + int num) { + struct token tok = {.type = type}; + if (type == words) { + tok.value = value; + } else if (type == number || type == boolean) { + tok.num = num; + } + if (null == array_list_append_token_ptr(token_list, &tok)) { + fprintf(stderr, "failed to append new token with id %s, %s", + token_type_to_string(type), cast(char *, value)); + exit(1); + } +} + +int isnumber(int c) { return '0' <= c && '9' >= c; } + +struct array_list_token *tokens_from_json_string(char *json) { + struct array_list_token *token_list = array_list_create_token(); + if (null == token_list) { + return null; + } + + struct string_view *sv = string_view_create(json); + + int curr; + while ((curr = string_view_next_char(sv)) != '\0') { + switch (curr) { + case '{': + append_token_to_token_list_or_fail(token_list, left_brace, null, 0); + break; + case '}': + append_token_to_token_list_or_fail(token_list, right_brace, null, 0); + break; + case '[': + append_token_to_token_list_or_fail(token_list, left_square_bracket, null, + 0); + break; + case ']': + append_token_to_token_list_or_fail(token_list, right_square_bracket, null, + 0); + break; + case '"': + append_token_to_token_list_or_fail(token_list, quote, null, 0); + break; + case ':': + append_token_to_token_list_or_fail(token_list, colon, null, 0); + break; + case 't': + if (0 == string_view_compare_word(sv, "rue")) { + append_token_to_token_list_or_fail(token_list, boolean, null, 1); + break; + } + goto default_case; + case 'f': + if (0 == string_view_compare_word(sv, "alse")) { + append_token_to_token_list_or_fail(token_list, boolean, null, 0); + break; + } + goto default_case; + case 'n': + if (0 == string_view_compare_word(sv, "ull")) { + append_token_to_token_list_or_fail(token_list, nullish, null, 0); + break; + } + goto default_case; + case ',': + append_token_to_token_list_or_fail(token_list, comma, null, 0); + break; + case '\n': + append_token_to_token_list_or_fail(token_list, newline, null, 0); + break; + case ' ': + case '\t': + append_token_to_token_list_or_fail(token_list, space, null, 0); + break; + default: + default_case: + if (isnumber(curr)) { + int num = curr - '0'; + int c; + while ((c = string_view_peek_char(sv)) != '\0' && isnumber(c)) { + cast(void, string_view_next_char(sv)); + num *= 10; + num += (c - '0'); + } + append_token_to_token_list_or_fail(token_list, number, null, num); + break; + } else { + const char *word = string_view_consume_word(sv); + char *word2 = string_create_empty(strlen(word) + 1); + word2[0] = curr; + strncpy(word2 + 1, word, strlen(word)); + append_token_to_token_list_or_fail(token_list, words, word2, 0); + break; + } + } + } + + append_token_to_token_list_or_fail(token_list, eof, null, 0); + + return token_list; +} + +int token_get_number(struct token *tok) { return tok->num; } +char *token_get_words(struct token *tok) { return tok->value; } +int token_get_bool(struct token *tok) { return tok->num; } diff --git a/libs/json/lexer_test.c b/libs/json/lexer_test.c new file mode 100644 index 0000000..b0c00bd --- /dev/null +++ b/libs/json/lexer_test.c @@ -0,0 +1,278 @@ +#define CISASTOR_TESTING_HALT_ON_FAIL 1 +#include "containers/array_list.h" +#include "containers/iterator.h" +#include "testing.h" +#include "token.h" +#include + +TEST(should_break_down_to_tokens) { + char *json = "{ \"id\": 1, \"active\": true, \"scores\": [10, 20], \"meta\": " + "{ \"verified\": null } }"; + struct array_list_token *tokens = tokens_from_json_string(json); + cassert(tokens != NULL, "failed to tokenize json string"); + + struct iterator *it = iterator_begin((struct array_list *)tokens); + + // { + cassert(iterator_element_token(it)->type == left_brace, "expected {"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // id + cassert(iterator_element_token(it)->type == words, "expected words 'id'"); + cassert(strcmp(token_get_words(iterator_element_token(it)), "id") == 0, + "expected 'id'"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // : + cassert(iterator_element_token(it)->type == colon, "expected :"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // 1 + cassert(iterator_element_token(it)->type == number, "expected number 1"); + cassert(token_get_number(iterator_element_token(it)) == 1, "expected 1"); + iterator_increment(it); + + // , + cassert(iterator_element_token(it)->type == comma, "expected ,"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // active + cassert(iterator_element_token(it)->type == words, "expected words 'active'"); + cassert(strcmp(token_get_words(iterator_element_token(it)), "active") == 0, + "expected 'active'"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // : + cassert(iterator_element_token(it)->type == colon, "expected :"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // true + cassert(iterator_element_token(it)->type == boolean, "expected boolean true"); + cassert(token_get_bool(iterator_element_token(it)) == 1, "expected true"); + iterator_increment(it); + + // , + cassert(iterator_element_token(it)->type == comma, "expected ,"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // scores + cassert(iterator_element_token(it)->type == words, "expected words 'scores'"); + cassert(strcmp(token_get_words(iterator_element_token(it)), "scores") == 0, + "expected 'scores'"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // : + cassert(iterator_element_token(it)->type == colon, "expected :"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // [ + cassert(iterator_element_token(it)->type == left_square_bracket, + "expected ["); + iterator_increment(it); + + // 10 + cassert(iterator_element_token(it)->type == number, "expected number 10"); + cassert(token_get_number(iterator_element_token(it)) == 10, "expected 10"); + iterator_increment(it); + + // , + cassert(iterator_element_token(it)->type == comma, "expected ,"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // 20 + cassert(iterator_element_token(it)->type == number, "expected number 20"); + cassert(token_get_number(iterator_element_token(it)) == 20, "expected 20"); + iterator_increment(it); + + // ] + cassert(iterator_element_token(it)->type == right_square_bracket, + "expected ]"); + iterator_increment(it); + + // , + cassert(iterator_element_token(it)->type == comma, "expected ,"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // meta + cassert(iterator_element_token(it)->type == words, "expected words 'meta'"); + cassert(strcmp(token_get_words(iterator_element_token(it)), "meta") == 0, + "expected 'meta'"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // : + cassert(iterator_element_token(it)->type == colon, "expected :"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // { + cassert(iterator_element_token(it)->type == left_brace, "expected {"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // verified + cassert(iterator_element_token(it)->type == words, + "expected words 'verified'"); + cassert(strcmp(token_get_words(iterator_element_token(it)), "verified") == 0, + "expected 'verified'"); + iterator_increment(it); + + // " + cassert(iterator_element_token(it)->type == quote, "expected \""); + iterator_increment(it); + + // : + cassert(iterator_element_token(it)->type == colon, "expected :"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // null + cassert(iterator_element_token(it)->type == nullish, "expected null"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // } + cassert(iterator_element_token(it)->type == right_brace, "expected }"); + iterator_increment(it); + + // space + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + // } + cassert(iterator_element_token(it)->type == right_brace, "expected }"); + iterator_increment(it); + + cassert(iterator_element_token(it)->type == eof, "expected eof"); + iterator_increment(it); + + cassert(iterator_element(it) == iterator_end(it), "expected end of tokens"); +} + +TEST(should_handle_empty_object) { + char *json = "{}"; + struct array_list_token *tokens = tokens_from_json_string(json); + cassert(tokens != NULL, "failed to tokenize empty object"); + struct iterator *it = iterator_begin((struct array_list *)tokens); + cassert(iterator_element_token(it)->type == left_brace, "expected {"); + iterator_increment(it); + cassert(iterator_element_token(it)->type == right_brace, "expected }"); + iterator_increment(it); + cassert(iterator_element_token(it)->type == eof, "expected eof"); + iterator_increment(it); + cassert(iterator_element(it) == iterator_end(it), "expected end"); +} + +TEST(should_handle_numbers) { + char *json = "12345"; + struct array_list_token *tokens = tokens_from_json_string(json); + struct iterator *it = iterator_begin((struct array_list *)tokens); + cassert(iterator_element_token(it)->type == number, "expected number"); + cassert(token_get_number(iterator_element_token(it)) == 12345, + "expected 12345"); +} + +TEST(should_handle_booleans_and_null) { + char *json = "true false null"; + struct array_list_token *tokens = tokens_from_json_string(json); + struct iterator *it = iterator_begin((struct array_list *)tokens); + + cassert(iterator_element_token(it)->type == boolean, "expected true"); + cassert(token_get_bool(iterator_element_token(it)) == 1, "expected 1"); + iterator_increment(it); + + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + cassert(iterator_element_token(it)->type == boolean, "expected false"); + cassert(token_get_bool(iterator_element_token(it)) == 0, "expected 0"); + iterator_increment(it); + + cassert(iterator_element_token(it)->type == space, "expected space"); + iterator_increment(it); + + cassert(iterator_element_token(it)->type == nullish, "expected null"); + iterator_increment(it); + + cassert(iterator_element_token(it)->type == eof, "expected eof"); + iterator_increment(it); +} diff --git a/libs/json/parser.c b/libs/json/parser.c new file mode 100644 index 0000000..34571b5 --- /dev/null +++ b/libs/json/parser.c @@ -0,0 +1,73 @@ +#include "arena.h" +#include "containers/array_list.h" +#include "containers/iterator.h" +#include "containers/stack.h" +#include "containers/string.h" +#include "containers/string_builder.h" +#include "json.h" +#include "token.h" + +#define to_base_array_list(al) cast(struct array_list *, al) + +struct token *next_non_empty_token(struct iterator *it) { + for_each_it(it) { + struct token *token = iterator_element_token(it); + if (token->type != space) { + return token; + } + } + return null; +} + +// state works through declaring the grammer in stages. +// state a parser is in, is not a single instance of enum. +// It's multiple states, recursive. +// Therefore, our state is a queue_state, advancing which, involves a next state +// and a finish state, which sets current state to the post-pop of the queue. + +string *parse_key(struct iterator *it) { return null; } + +struct json_value *parse_string(struct iterator *it) { + string_builder *sb = string_builder_create(); + for_each_it(it) { + struct token *token = iterator_element_token(it); + if (token->type == quote) { + break; + } + switch (token->type) { + case words: + string_builder_join(sb, token->value); + break; + case space: + string_builder_join(sb, " "); + break; + case eof: + fprintf(stderr, "unexpected EOF while reading string"); + return null; + + default: + fprintf(stderr, "expected character stream, got '%s'\n", + token_to_string(token)); + return null; + } + } + string *str = string_builder_build(sb); + struct json_value *jv = aalloc(sizeof(struct json_value)); + if (jv == null) { + return null; + } + jv->kind = json_string; + jv->string = str; + return jv; +} + +struct json_value *parse_list(struct iterator *it) { return null; } + +struct json_value *parse_object(struct iterator *it) { return null; } + +struct json_value *parse(struct array_list_token *al) { + for_each(it, to_base_array_list(al)) { + struct token *token = next_non_empty_token(it); + } + return null; +} diff --git a/libs/json/parser.h b/libs/json/parser.h new file mode 100644 index 0000000..8fc79a2 --- /dev/null +++ b/libs/json/parser.h @@ -0,0 +1,7 @@ +#pragma once + +#include "containers/array_list.h" +#include "token.h" + +struct json_value *parse(struct array_list_token *al); + diff --git a/libs/json/parser_test.c b/libs/json/parser_test.c new file mode 100644 index 0000000..7129b8e --- /dev/null +++ b/libs/json/parser_test.c @@ -0,0 +1,99 @@ +#include "containers/array_list.h" +#include "containers/iterator.h" +#include "containers/string.h" +#include "json.h" +#include "testing.h" +#include "token.h" +#include + +static uint8_t string_equals_cstr(string *s, const char *cstr) { + return string_cmp_cstr(s, cstr) == 0; +} + +IT(should_parse_empty_string) { + struct array_list_token *al = array_list_create_token(); + token t1 = {.type = quote}; + array_list_append_token(al, t1); + + struct iterator *it = iterator_begin((struct array_list *)al); + struct json_value *jv = parse_string(it); + + cassert(jv != NULL, "jv != NULL"); + cassert(jv->kind == json_string, "kind == json_string"); + cassert(string_length(jv->string) == 0, "length == 0"); + cassert(iterator_element_token(it)->type == quote, "at quote"); +} + +IT(should_parse_string_with_words) { + struct array_list_token *al = array_list_create_token(); + token t1 = {.type = words, .value = "hello"}; + token t2 = {.type = quote}; + array_list_append_token(al, t1); + array_list_append_token(al, t2); + + struct iterator *it = iterator_begin((struct array_list *)al); + struct json_value *jv = parse_string(it); + + cassert(jv != NULL, "jv != NULL"); + cassert(string_equals_cstr(jv->string, "hello"), "equals hello"); + cassert(iterator_element_token(it)->type == quote, "at quote"); +} + +IT(should_parse_string_with_spaces) { + struct array_list_token *al = array_list_create_token(); + token t1 = {.type = words, .value = "a"}; + token t2 = {.type = space}; + token t3 = {.type = words, .value = "b"}; + token t4 = {.type = quote}; + array_list_append_token(al, t1); + array_list_append_token(al, t2); + array_list_append_token(al, t3); + array_list_append_token(al, t4); + + struct iterator *it = iterator_begin((struct array_list *)al); + struct json_value *jv = parse_string(it); + + cassert(jv != NULL, "jv != NULL"); + cassert(string_equals_cstr(jv->string, "a b"), "equals 'a b'"); + cassert(iterator_element_token(it)->type == quote, "at quote"); +} + +IT(should_parse_string_with_only_spaces) { + struct array_list_token *al = array_list_create_token(); + token t1 = {.type = space}; + token t2 = {.type = space}; + token t3 = {.type = quote}; + array_list_append_token(al, t1); + array_list_append_token(al, t2); + array_list_append_token(al, t3); + + struct iterator *it = iterator_begin((struct array_list *)al); + struct json_value *jv = parse_string(it); + + cassert(jv != NULL, "jv != NULL"); + cassert(string_equals_cstr(jv->string, " "), "equals ' '"); + cassert(iterator_element_token(it)->type == quote, "at quote"); +} + +IT(should_parse_string_with_mixed_words_and_spaces) { + struct array_list_token *al = array_list_create_token(); + token t1 = {.type = space}; + token t2 = {.type = words, .value = "foo"}; + token t3 = {.type = space}; + token t4 = {.type = words, .value = "bar"}; + token t5 = {.type = space}; + token t6 = {.type = quote}; + array_list_append_token(al, t1); + array_list_append_token(al, t2); + array_list_append_token(al, t3); + array_list_append_token(al, t4); + array_list_append_token(al, t5); + array_list_append_token(al, t6); + + struct iterator *it = iterator_begin((struct array_list *)al); + struct json_value *jv = parse_string(it); + + cassert(jv != NULL, "jv != NULL"); + cassert(string_equals_cstr(jv->string, " foo bar "), "equals ' foo bar '"); + cassert(iterator_element_token(it)->type == quote, "at quote"); +} diff --git a/libs/json/token.h b/libs/json/token.h new file mode 100644 index 0000000..0a4ae55 --- /dev/null +++ b/libs/json/token.h @@ -0,0 +1,39 @@ +#pragma once + +#include "containers/array_list.h" +enum token_type { + left_brace, + quote, + colon, + words, + number, + boolean, + nullish, + comma, + right_brace, + newline, + space, + left_square_bracket, + right_square_bracket, + eof, + + _token_types_count, +}; + +typedef struct token { + enum token_type type; + union { + char *value; + int num; + }; +} token; + +array_list_init(token); + +int token_get_number(struct token *tok); +char *token_get_words(struct token *tok); +int token_get_bool(struct token *tok); + +struct array_list_token *tokens_from_json_string(char *json); + +char *token_to_string(struct token *tok); diff --git a/libs/progressbar/progressbar.c b/libs/progressbar/progressbar.c index 7a64e3c..594a1ed 100644 --- a/libs/progressbar/progressbar.c +++ b/libs/progressbar/progressbar.c @@ -4,6 +4,8 @@ * A ProgressBar is a drop in replacement of std::ostream; */ +#define _GNU_SOURCE + #include "macros.h" #include diff --git a/libs/utils/macros.h b/libs/utils/macros.h index 83f85c7..bdaca8d 100644 --- a/libs/utils/macros.h +++ b/libs/utils/macros.h @@ -7,5 +7,12 @@ #define byte char -#endif +// intentionally left without formatting allowed, if need formatting, just log +// as is. +#define unreachable(str) \ + do { \ + fprintf(stderr, "unrechable block hit, diagnostic message: %s\n", str); \ + abort(); \ + } while (0) +#endif