feat: Add error handling and tests for unknown keywords

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: Run Tests
+name: Run DogLang Tests
 
 on:
   pull_request:

doglang/SyntaxAnalyser.py

@@ -1,5 +1,7 @@
-from doglang.Tokenizer import Tokens, Tokenizer
+from doglang.Tokenizer import Tokens
 from doglang.SymbolTable import SymbolTable
+# Import the custom exception we created in error.py
+from doglang.error import DogLangSyntaxError
 
 class AST:
     def __init__(self,type,value=None):
@@ -27,10 +29,6 @@ def _pretty_print(self, indent=0):
         result += " " * indent + "])"
         return result
 
-class Error:
-    def __init__(self,error):
-        raise Exception(f'Syntax Error: {error}')
-
 class SyntaxAnalyser(SymbolTable):
     def __init__(self,token) -> None:
         self.token=token
@@ -40,42 +38,59 @@ def current_element(self):
         if self.current < len(self.token):
             return self.token[self.current]
         return None
+
     def increment(self):
         self.current+=1
 
+    # Add a 'peek' method to look at the next token without consuming it
+    def peek(self):
+        if self.current + 1 < len(self.token):
+            return self.token[self.current + 1]
+        return None
+
     def match(self,expected_type,expected_value=None):
         token=self.current_element()
         if not token:
-            Error(f"Unexpected end of input. Expected {expected_type}")
+            raise DogLangSyntaxError(f"Unexpected end of input. Expected {expected_type}")
         if token.token_type != expected_type or (expected_value is not None and token.value != expected_value):
-            Error(f"Expected {expected_type},{expected_value} but got {token}")
+            # Use the new error class and include the line number
+            raise DogLangSyntaxError(f"Expected {expected_type} '{expected_value}' but got {token.token_type} '{token.value}' at line {token.line}")
         self.increment()
         return token
 
-
     #Grammar rules
-
     def parse(self):
         return self.program()
+
     def program(self):
         node=AST("Program")
-        while self.current<len(self.token):
+        while self.current < len(self.token):
             node.addchild(self.statement())
         return node
 
+
     def statement(self):
         token=self.current_element()
 
-        if token.token_type == Tokens.KEYWORD and token.value=='bark':
-            return self.print_stmt()
+        if token.token_type == Tokens.KEYWORD:
+            if token.value=='bark':
+                return self.print_stmt()
+            elif token.value=='wagtail':
+                return self.loop_stmt()
+            elif token.value=='sniff':
+                return self.conditional_statement()
+
         elif token.token_type == Tokens.IDENTIFIER:
-            return self.assignment()
-        elif token.token_type == Tokens.KEYWORD and token.value=='wagtail':
-            return self.loop_stmt()
-        elif token.token_type == Tokens.KEYWORD and token.value=='sniff':
-            return self.conditional_statement()
-        else:
-            Error("Unexpected Token.")
+            # Look ahead to see if the next token is an assignment operator
+            next_token = self.peek()
+            if next_token and next_token.token_type == Tokens.ASSIGNMENT_OP:
+                return self.assignment()
+            else:
+                # If it's an identifier NOT followed by '=', it's an unknown keyword
+                raise DogLangSyntaxError(f"Syntax Error: Unknown keyword '{token.value}' at line {token.line}")
+
+        # If we reach here, the token is not a valid start to a statement
+        raise DogLangSyntaxError(f"Unexpected token '{token.value}' at line {token.line}")
 
     def loop_stmt(self):
         node=AST("loop")
@@ -85,17 +100,15 @@ def loop_stmt(self):
         while self.current_element().value != '}':
             node.addchild(self.statement())
         self.match(Tokens.CURLY_BRACE,'}')
-
         return node
 
     def assignment(self): 
         node=AST("assignment")
-        id = self.current_element().value #To get identifier name
+        id = self.current_element().value
         node.addchild(AST(Tokens.IDENTIFIER,id))
-        self.match(Tokens.IDENTIFIER) # identifier
-        self.match(Tokens.ASSIGNMENT_OP,'=')  #checks for = 
+        self.match(Tokens.IDENTIFIER)
+        self.match(Tokens.ASSIGNMENT_OP,'=')
         node.addchild(self.expressions())
-
         return node
 
     def code_block(self):
@@ -114,6 +127,7 @@ def conditional_statement(self):
         if(self.current_element() and self.current_element().value == 'else'):
             node.addchild(self.else_statement())
         return node
+
     def else_statement(self):
         node = AST(Tokens.KEYWORD,"else")
         self.match(Tokens.KEYWORD)
@@ -128,30 +142,23 @@ def expressions(self):
                 self.match(Tokens.KEYWORD,'fetch')
                 node = AST(Tokens.KEYWORD,"input")
                 node.addchild(self.expressions())
+                return node 
 
-
-        if token.token_type == Tokens.INT_LITERAL or token.token_type == Tokens.PARENTHESIS or token.token_type == Tokens.IDENTIFIER:
-            node=AST("expression")
-            while self.current_element().value != ';':
-                if self.current_element().token_type == Tokens.CURLY_BRACE: 
-                    return node
-                node.addchild(AST(self.current_element().token_type,self.current_element().value))
-                self.increment()
-
+        node=AST("expression")
+        while self.current_element() and self.current_element().value != ';':
+            if self.current_element().token_type == Tokens.CURLY_BRACE: 
+                return node
+            node.addchild(AST(self.current_element().token_type,self.current_element().value))
             self.increment()
-
+
+        # Consume the semicolon if it exists
+        if self.current_element() and self.current_element().value == ';':
+            self.increment()
+
         return node
 
-
     def print_stmt(self):
         node=AST("print")
         self.match(Tokens.KEYWORD,'bark') #bark keyword
         node.addchild(self.expressions())
-
         return node
-
-
-
-
-
-

doglang/Tokenizer.py

@@ -1,13 +1,17 @@
 import re
+from .error import DogLangSyntaxError 
 
 
 class Token:
-    def __init__(self, token_type, value):
+
+    def __init__(self, token_type, value, line):
         self.token_type = token_type
         self.value = value
+        self.line = line
 
     def __repr__(self):
-        return f"\nToken({self.token_type}, '{self.value}')"
+
+        return f"\nToken({self.token_type}, '{self.value}', line={self.line})"
 
 
 class Tokens:
@@ -44,63 +48,40 @@ class Tokens:
 def Tokenizer(code):
     tokens = []
 
-
-    pattern = r'"(?:\\.|[^"\\])*"|[A-Za-z_]\w*|\d+|==|!=|>=|<=|&&|\|\||[+\-*/%]=?|[(){};,]|[<>]|='
-    tokenized_code = re.findall(pattern, code)
-
-    for word in tokenized_code:
-        # Check for string literals
-        if word.startswith('"') and word.endswith('"'):
-            tokens.append(Token(Tokens.STRING_LITERAL, word[1:-1]))  # Remove the quotes
-
-        # Check for keywords
-        elif word in keywords:
-            tokens.append(Token(Tokens.KEYWORD, word))
-
-        # Check for identifiers
-        elif word.isidentifier():
-            tokens.append(Token(Tokens.IDENTIFIER, word))
-
-        # Check for assignment operator
-        elif word == '=':
-            tokens.append(Token(Tokens.ASSIGNMENT_OP, word))
-
-        # Check for literals (numbers)
-        elif word.isdigit():
-            tokens.append(Token(Tokens.INT_LITERAL, word))
-
-        # Check for arithmetic operators
-        elif word in arithmetic_operators:
-            tokens.append(Token(Tokens.ARITHMETIC_OP, word))
-
-        # Check for comparison operators
-        elif word in comparison_operators:
-            tokens.append(Token(Tokens.COMPARISON_OP, word))
-
-        # Check for logical operators
-        elif word in logical_operators:
-            tokens.append(Token(Tokens.LOGICAL_OP, word))
-
-        # Check for parentheses
-        elif word in parentheses:
-            tokens.append(Token(Tokens.PARENTHESIS, word))
-
-        # Check for curly braces
-        elif word in curly_braces:
-            tokens.append(Token(Tokens.CURLY_BRACE, word))
-
-        # Check for separators like comma, period
-        elif word in separators:
-            tokens.append(Token(Tokens.SEPARATOR, word))
-
-        # Check for semicolon
-        elif word == semicolon:
-            tokens.append(Token(Tokens.SEMICOLON, word))
-
-
-        else:
-            print(f"Unrecognized token: {word}")
-
+    # Process the code line by line to get the line number
+    for line_number, line in enumerate(code.splitlines(), 1):
+        pattern = r'"(?:\\.|[^"\\])*"|[A-Za-z_]\w*|\d+|==|!=|>=|<=|&&|\|\||[+\-*/%]=?|[(){};,]|[<>]|='
+        tokenized_line = re.findall(pattern, line)
+
+        for word in tokenized_line:
+            # Pass the 'line_number' to every new Token instance
+            if word.startswith('"') and word.endswith('"'):
+                tokens.append(Token(Tokens.STRING_LITERAL, word[1:-1], line_number))
+            elif word in keywords:
+                tokens.append(Token(Tokens.KEYWORD, word, line_number))
+            elif word.isidentifier():
+                tokens.append(Token(Tokens.IDENTIFIER, word, line_number))
+            elif word == '=':
+                tokens.append(Token(Tokens.ASSIGNMENT_OP, word, line_number))
+            elif word.isdigit():
+                tokens.append(Token(Tokens.INT_LITERAL, word, line_number))
+            elif word in arithmetic_operators:
+                tokens.append(Token(Tokens.ARITHMETIC_OP, word, line_number))
+            elif word in comparison_operators:
+                tokens.append(Token(Tokens.COMPARISON_OP, word, line_number))
+            elif word in logical_operators:
+                tokens.append(Token(Tokens.LOGICAL_OP, word, line_number))
+            elif word in parentheses:
+                tokens.append(Token(Tokens.PARENTHESIS, word, line_number))
+            elif word in curly_braces:
+                tokens.append(Token(Tokens.CURLY_BRACE, word, line_number))
+            elif word in separators:
+                tokens.append(Token(Tokens.SEPARATOR, word, line_number))
+            elif word == semicolon:
+                tokens.append(Token(Tokens.SEMICOLON, word, line_number))
+            else:
+                # Raise a syntax error for unrecognized tokens
+                raise DogLangSyntaxError(f"Unrecognized token '{word}' at line {line_number}")
 
     return tokens
 

doglang/__init__.py

@@ -1,5 +1,5 @@
 from doglang.main import Interpreter
-from doglang.error import Error
+from doglang.error import DogLangError
 
 __version__ = "1.0.0-alpha"
 __all__ = ["Interpreter", "Error"]

doglang/error.py

@@ -1,3 +1,7 @@
-class Error:
-    def __init__(self,stage,error):
-        raise Exception(f'{stage} Error: {error}')
+class DogLangError(Exception):
+    """Base exception class for all errors in DogLang."""
+    pass
+
+class DogLangSyntaxError(DogLangError):
+    """Exception raised for syntax errors found during parsing."""
+    pass

doglang/main.py

@@ -2,7 +2,7 @@
 from doglang.SyntaxAnalyser import SyntaxAnalyser
 from doglang.Tokenizer import Tokenizer
 from doglang.SemanticAnalyser import SemanticAnalyser
-from doglang.error import Error
+from doglang.error import DogLangError
 
 class Interpreter:
     def __init__(self,code):
@@ -52,7 +52,7 @@ def conditions(self,children):
                              if len(children) > 2:
                                 self.visit(children[2].children[0])
                    else:
-                        Error("Type","Value inside sniff is not boolean.")
+                        DogLangError("Type","Value inside sniff is not boolean.")
 
     def print_stmt(self,children):
             for child in children:

tests/test_syntax_analyser.py

@@ -0,0 +1,40 @@
+import pytest
+from doglang.Tokenizer import Tokenizer
+from doglang.SyntaxAnalyser import SyntaxAnalyser
+from doglang.error import DogLangSyntaxError
+
+def test_parser_handles_valid_code():
+    """Tests that a simple, valid program parses without errors."""
+    code = 'bark "hello";'
+    tokens = Tokenizer(code)
+    analyser = SyntaxAnalyser(tokens)
+    # This should run without raising any exceptions
+    analyser.parse()
+
+def test_parser_raises_error_for_unknown_keyword():
+    """
+    This is the main test.
+    It checks that a typo like 'wagdog' raises a specific syntax error.
+    """
+    # This code contains an invalid keyword on the second line
+    code_with_typo = 'bark "hello";\nwagdog'
+
+    tokens = Tokenizer(code_with_typo)
+    analyser = SyntaxAnalyser(tokens)
+
+    # Use pytest.raises to check that the correct exception is thrown
+    with pytest.raises(DogLangSyntaxError) as excinfo:
+        analyser.parse() # This line should trigger the error
+
+    # Assert that the error message is exactly what the issue requires
+    assert str(excinfo.value) == "Syntax Error: Unknown keyword 'wagdog' at line 2"
+
+def test_parser_raises_error_for_unexpected_token():
+    """Tests that the parser fails on other invalid syntax."""
+    code_with_bad_token = '"hello";' # A string cannot start a statement
+
+    tokens = Tokenizer(code_with_bad_token)
+    analyser = SyntaxAnalyser(tokens)
+
+    with pytest.raises(DogLangSyntaxError):
+        analyser.parse()