a tokenizer without 'type' and 'loc'

Author: ppaska

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "chrome",
+            "request": "launch",
+            "name": "Launch Chrome against localhost",
+            "url": "http://localhost:10001",
+            "webRoot": "${workspaceFolder}"
+        }
+    ]
+}

index.html

@@ -29,8 +29,9 @@
   <div class="container">
     <h4>JSPython development console</h4>
     <div id="editor">a = 1
-b = a+2*3
+      b = a+2*3
     </div>
+    <button onclick="tokenize()">Tokenize</button>
     <button onclick="parse()">Parse</button>
     <button onclick="runInterpreter()">Run</button>
     <textarea id="result"></textarea>
@@ -42,6 +43,23 @@ <h4>JSPython development console</h4>
     editor.session.setMode("ace/mode/python");
 
     const jsPython = jspython.jsPython;
+    function tokenize() {
+      tokenizer = (s) =>  console.log(`tokens => ${s}`, jsPython().tokenize(s))
+
+      tokenizer("x='hello' + ' ' + 'world'")
+
+      const scripts = editor.getValue();
+      try {
+        const result = jsPython()
+          .tokenize(scripts).map(t => t[0]);
+
+        document.getElementById('result').value = typeof result === 'object' ? JSON.stringify(result, null, '\t') : result
+        console.log('Result => ', result);
+      } catch (err) {
+        document.getElementById('result').value = err;
+        console.error(err);
+      }
+    }
 
     async function parse() {
 

src/interpreter.ts

@@ -1,4 +1,4 @@
-import { Ast } from './common';
+import { Ast, Token } from './common';
 import { Evaluator } from './evaluator';
 import { Parser } from './parser';
 import { Tokenizer } from './tokenizer';
@@ -17,6 +17,11 @@ export class Interpreter {
         return new Interpreter();
     }
 
+    tokenize(script: string): Token[] {
+        const tokenizer = new Tokenizer();
+        return tokenizer.tokenize(script);
+    }
+
     parse(script: string): Ast {
         const tokenizer = new Tokenizer();
         const parser = new Parser();

src/tokenizer/tokenizer.spec.ts

@@ -0,0 +1,93 @@
+import { Tokenizer } from "./tokenizer";
+
+describe('Tokenizer => ', () => {
+
+
+  beforeEach(() => {});
+
+  it('a + b + 55', async () => {
+    let tokens = new Tokenizer().tokenize("a + b + 55")
+    expect(tokens.length).toBe(5);
+    tokens = new Tokenizer().tokenize("a+b+55")
+    expect(tokens.length).toBe(5);
+  });
+
+  it('s = 255 + 23 * 45', async () => {
+    let tokens = new Tokenizer().tokenize("s = 255 + 23 * 45")
+    expect(tokens.length).toBe(7);
+    tokens = new Tokenizer().tokenize("s =255+23*45")
+    expect(tokens.length).toBe(7);
+  });
+
+  it('s=(255 + 23) * 45', async () => {
+    let tokens = new Tokenizer().tokenize("s = (255 + 23 ) * 45")
+    expect(tokens.length).toBe(9);
+    tokens = new Tokenizer().tokenize("s=(255 + 23) * 45")
+    expect(tokens.length).toBe(9);
+    tokens = new Tokenizer().tokenize("s=(255 \n      +\n    23) \n   *    45")
+    expect(tokens.length).toBe(9);
+  });
+
+
+  it('if someVar == 20/40:\n  someVar = 55', async () => {
+    let tokens = new Tokenizer().tokenize("if someVar == 20/40:\n  someVar = 55")
+    expect(tokens.length).toBe(10);
+    tokens = new Tokenizer().tokenize("if someVar== 20/40:\n  someVar=55")
+    expect(tokens.length).toBe(10);
+    tokens = new Tokenizer().tokenize("if someVar==20/40:\n    someVar= 55")
+    expect(tokens.length).toBe(10);
+  });
+
+  it('x="test1"', async () => {
+    let tokens = new Tokenizer().tokenize('x="test1"')
+    expect(tokens.length).toBe(3);
+    expect(tokens[2][0]).toBe('test1');
+    tokens = new Tokenizer().tokenize('x ="test1" ')
+    expect(tokens.length).toBe(3);
+    expect(tokens[2][0]).toBe('test1');
+    tokens = new Tokenizer().tokenize('x="test1" ')
+    expect(tokens.length).toBe(3);
+    expect(tokens[2][0]).toBe('test1');
+  });
+
+  it('x="hello" + " " + "world"', async () => {
+    let tokens = new Tokenizer().tokenize('x="hello"+" "+"world"')
+    expect(tokens.length).toBe(7);
+    expect(tokens[2][0]).toBe('hello');
+    expect(tokens[4][0]).toBe(' ');
+    expect(tokens[6][0]).toBe('world');
+
+    tokens = new Tokenizer().tokenize('x="hello" + " "+"world"')
+    expect(tokens.length).toBe(7);
+    expect(tokens[2][0]).toBe('hello');
+    expect(tokens[4][0]).toBe(' ');
+    expect(tokens[5][0]).toBe('+');
+    expect(tokens[6][0]).toBe('world');
+    tokens = new Tokenizer().tokenize("x='hello' + ' ' + 'world'")
+    expect(tokens.length).toBe(7);
+    expect(tokens[2][0]).toBe('hello');
+    expect(tokens[4][0]).toBe(' ');
+    expect(tokens[6][0]).toBe('world');
+  });
+
+  it('x=""', async () => {
+    let tokens = new Tokenizer().tokenize('x=""')
+    expect(tokens.length).toBe(3);
+    expect(tokens[2][0]).toBe('');
+  });
+
+  it('x="" # this is comment', async () => {
+    let tokens = new Tokenizer().tokenize('x="" # this is comment')
+    expect(tokens.length).toBe(4);
+    expect(tokens[3][0]).toBe(' this is comment');
+  });
+
+  it('x= # this is comment \n 5+6', async () => {
+    let tokens = new Tokenizer().tokenize('x= # this is comment \n 5+6')
+    expect(tokens.length).toBe(6);
+    expect(tokens[4][0]).toBe('+');
+  });
+
+
+  
+});

src/tokenizer/tokenizer.ts

@@ -1,13 +1,112 @@
-import { Token } from "../common";
+import { Token, TokenTypes } from "../common";
 
-export class Tokenizer
-{
+const SeparatorsMap: Record<string, string[]> = {
+    '=': ['=', '==', '=>'],
+
+    '+': ['+', '++', '+='],
+    '-': ['-', '--', '-='],
+    '*': ['*', '**', '*='],
+    '/': ['/', '//', '/='],
+
+    '.': ['.'],
+    '?': ['?'],
+    '!': ['!='],
+    ':': [':'],
+    ',': [','],
+
+    '>': ['>', '>='],
+    '<': ['<', '<=', '<>'],
+
+    '(': ['('],
+    ')': [')'],
+    '{': ['{'],
+    '}': ['}'],
+    '[': ['['],
+    ']': [']'],
+};
+
+export class Tokenizer {
+    private recognizeToken(tokenText: string): { value: string | number | boolean, type: TokenTypes } {
+        return {
+            value: tokenText,
+            type: TokenTypes.LiteralString
+        }
+
+    }
+    private processToken(strToken: string, tokens: Token[], allowEmptyString = false): string {
+        // ignore empty tokens
+        if(!strToken.length && !allowEmptyString) return "";
+        
+        var token = this.recognizeToken(strToken);
+        tokens.push([token.value, Uint16Array.of(token.type as number, 0, 0, 0, 0)] as Token)
+        return "";
+    }    
     /**
      * Splits script code into a tokens
      * @param script A jsPython text
      */
-    tokenize(script: string): Token[]  {
-        return [];        
+    tokenize(script: string): Token[] {
+        if (!script || !script.length) { return []; }
+
+        let cursor = 0;
+        const tokens: Token[] = [];
+        let tokenText = "";
+        let currentLine = 1;
+        let currentColumn = 1;
+
+        do {
+            let symbol = script[cursor]
+            currentColumn++;
+            if (symbol == '\n') {
+                currentLine++;
+                currentColumn = 1
+                continue;
+            } else if (symbol == ' ' && tokenText.length !== 0) {
+                tokenText = this.processToken(tokenText, tokens);
+                continue;
+            } else if (SeparatorsMap[symbol] !== undefined) {                
+                tokenText = this.processToken(tokenText, tokens);
+                tokenText = symbol;
+
+                const sepsMap = SeparatorsMap[symbol];
+
+                if (sepsMap.length > 1) {
+                    // process longer operators
+                    while (sepsMap.includes(tokenText + script[cursor + 1])) {
+                        tokenText += script[++cursor]; 
+                    }
+                } 
+                tokenText = this.processToken(tokenText, tokens);
+
+            }  else if (symbol === '#') {
+
+                while(script[++cursor] !== '\n') {
+                    tokenText += script[cursor];
+                    if(cursor + 1 >= script.length) break;
+                }
+                tokenText = this.processToken(tokenText, tokens, true);
+
+            } else if (symbol === '"' || symbol === "'") {
+                // remember either it is single or double quote
+                const q = symbol;
+                // we are not expecting token to be added here.
+                // it should pass a failt to parser
+                tokenText = this.processToken(tokenText, tokens);
+
+                while(script[++cursor] !== q) {
+                    tokenText += script[cursor];
+                    if(cursor + 1 >= script.length) break;
+                }
+                tokenText = this.processToken(tokenText, tokens, true);
+            } else if (symbol != ' ') {
+                tokenText += symbol;
+            }
+        }
+        while (++cursor < script.length)
+
+        this.processToken(tokenText, tokens);
+
+        return tokens;
     }
 
 }