Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 6645b2c

Browse files
ppaskappaska
committedDec 2, 2020
added """ strings
1 parent a538fc8 commit 6645b2c

File tree

5 files changed

+62
-25
lines changed

5 files changed

+62
-25
lines changed
 

‎.eslintrc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
"@typescript-eslint/no-explicit-any": "warn",
1414
"@typescript-eslint/explicit-module-boundary-types": "warn",
1515
"@typescript-eslint/explicit-function-return-type": "warn",
16-
"@typescript-eslint/no-unused-vars": "off"
16+
"@typescript-eslint/no-unused-vars": "off",
17+
"@typescript-eslint/no-control-regex": "off",
18+
"@typescript-eslint/no-constant-condition": "off"
1719
}
1820
}

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,4 @@ Thumbs.db
4343

4444
package-lock.json
4545
*.tgz
46+
/*.log

‎index.html

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@
1313
margin: 10px auto;
1414
}
1515

16-
#editor,
17-
#result {
16+
#editor,
17+
#resultEditor {
1818
height: 40%;
1919
display: block;
2020
min-height: 20%;
@@ -28,19 +28,25 @@
2828
<body>
2929
<div class="container">
3030
<h4>JSPython development console</h4>
31-
<div id="editor">x=3.14
31+
<div id="editor">x=3.1433
3232
</div>
3333
<button onclick="tokenize()">Tokenize</button>
3434
<button onclick="parse()">Parse</button>
3535
<button onclick="runInterpreter()">Run</button>
36-
<textarea id="result"></textarea>
36+
<div id="resultEditor"> </div>
37+
38+
<!-- <textarea id="result"></textarea> -->
3739
</div>
3840
<script>
3941

4042
const editor = ace.edit("editor");
4143
editor.setTheme("ace/theme/monokai");
4244
editor.session.setMode("ace/mode/python");
4345

46+
const resultEditor = ace.edit("resultEditor");
47+
resultEditor.setTheme("ace/theme/monokai");
48+
resultEditor.session.setMode("ace/mode/json");
49+
4450
const jsPython = jspython.jsPython;
4551
function tokenize() {
4652
tokenizer = (s) => console.log(`tokens => ${s}`, jsPython().tokenize(s))
@@ -50,13 +56,16 @@ <h4>JSPython development console</h4>
5056
const scripts = editor.getValue();
5157
try {
5258
const result = jsPython()
53-
.tokenize(scripts).map(t => t[0]);
59+
.tokenize(scripts)
60+
.map(t => `${t[1]} : '${t[0]}'`)
61+
.join('\n');
5462

55-
document.getElementById('result').value = typeof result === 'object' ? JSON.stringify(result, null, '\t') : result
56-
console.log('Result => ', result);
63+
// document.getElementById('result').value = typeof result === 'object' ? JSON.stringify(result, null, '\t') : result
64+
const data = typeof result === 'object' ? JSON.stringify(result, null, '\t') : result;
65+
resultEditor.getSession().setValue(data)
5766
} catch (err) {
58-
document.getElementById('result').value = err;
5967
console.error(err);
68+
document.getElementById('result').value = err;
6069
}
6170
}
6271

@@ -70,8 +79,8 @@ <h4>JSPython development console</h4>
7079
document.getElementById('result').value = typeof result === 'object' ? JSON.stringify(result, null, '\t') : result
7180
console.log('Result => ', result);
7281
} catch (err) {
73-
document.getElementById('result').value = err;
7482
console.error(err);
83+
document.getElementById('result').value = err;
7584
}
7685
}
7786

@@ -85,8 +94,8 @@ <h4>JSPython development console</h4>
8594
document.getElementById('result').value = typeof result === 'object' ? JSON.stringify(result, null, '\t') : result
8695
console.log('Result => ', result);
8796
} catch (err) {
88-
document.getElementById('result').value = err;
8997
console.error(err);
98+
document.getElementById('result').value = err;
9099
}
91100
}
92101

‎src/tokenizer/tokenizer.spec.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,19 @@ describe('Tokenizer => ', () => {
101101
expect(tokens[4][0]).toBe(3.14);
102102
});
103103

104+
it('"""test 1"""', async () => {
105+
const tokens = new Tokenizer().tokenize('"""test 1"""')
106+
expect(tokens.length).toBe(1);
107+
expect(tokens[0][0]).toBe('test 1');
108+
});
109+
110+
it('x="""test 1"""+"d"', async () => {
111+
const tokens = new Tokenizer().tokenize('x="""test 1"""+"d"')
112+
expect(tokens.length).toBe(5);
113+
expect(tokens[2][0]).toBe('test 1');
114+
expect(tokens[4][0]).toBe('d');
115+
});
116+
104117
/*
105118
it('3 - -2', async () => {
106119
let tokens = new Tokenizer().tokenize('3 - -2')

‎src/tokenizer/tokenizer.ts

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Token, TokenTypes } from '../common';
22

33
const SeparatorsMap: Record<string, string[]> = {
4+
'\n': ['\n'],
45
'=': ['=', '==', '=>'],
56

67
'+': ['+', '++', '+='],
@@ -57,7 +58,7 @@ export class Tokenizer {
5758
}
5859
private processToken(strToken: string, tokens: Token[], allowEmptyString = false, type: TokenTypes | null = null): string {
5960
// ignore empty tokens
60-
if (!strToken.length && !allowEmptyString) return "";
61+
if (!strToken.length && !allowEmptyString || strToken === '\n') return "";
6162

6263
const token = this.recognizeToken(strToken, type);
6364
tokens.push([token.value, Uint16Array.of(token.type as number, 0, 0, 0, 0)] as Token)
@@ -90,7 +91,7 @@ export class Tokenizer {
9091
}
9192

9293
private isPartOfNumber(symbol: string, token: string, cursor: number): boolean {
93-
94+
// '-' needs to be handled e.g. -3; 2 + -2 etc
9495
// if(token.length == 0 && symbol === '-') {
9596
// return true;
9697
// }
@@ -106,23 +107,21 @@ export class Tokenizer {
106107
tokenize(script: string): Token[] {
107108
if (!script || !script.length) { return []; }
108109

110+
script = script
111+
.replace(new RegExp('\t', 'g'), ' ') // replace all tabs with 2 spaces
112+
.replace(new RegExp('\r', 'g'), ''); // remove all \r symbols
113+
109114
let cursor = 0;
110115
const tokens: Token[] = [];
111116
let tokenText = "";
112-
let currentLine = 1;
113-
let currentColumn = 1;
114117

115118
do {
116119
const symbol = script[cursor]
117-
currentColumn++;
118-
if (symbol == '\n') {
119-
currentLine++;
120-
currentColumn = 1;
121-
continue;
122-
} else if (symbol == ' ' && tokenText.length !== 0) {
120+
121+
if (symbol == ' ' && tokenText.length !== 0) {
123122
tokenText = this.processToken(tokenText, tokens);
124123
continue;
125-
} else if (SeparatorsMap[symbol] && !this.isPartOfNumber(symbol, tokenText, cursor)) {
124+
} else if ((SeparatorsMap[symbol]) && !this.isPartOfNumber(symbol, tokenText, cursor)) {
126125
// handle numbers with floating point e.g. 3.14
127126
tokenText = this.processToken(tokenText, tokens);
128127
tokenText = symbol;
@@ -152,9 +151,22 @@ export class Tokenizer {
152151
// it should pass a failt to parser
153152
tokenText = this.processToken(tokenText, tokens);
154153

155-
while (script[++cursor] !== q) {
156-
tokenText += script[cursor];
157-
if (cursor + 1 >= script.length) break;
154+
// handle """ comment """"
155+
if (script[cursor + 1] === q && script[cursor + 2] === q) {
156+
cursor += 2;
157+
while (true) {
158+
tokenText += script[++cursor];
159+
if (cursor + 3 >= script.length
160+
|| (script[cursor + 1] === q && script[cursor + 2] === q && script[cursor + 3] === q)) {
161+
break;
162+
}
163+
}
164+
cursor += 3;
165+
} else {
166+
while (script[++cursor] !== q) {
167+
tokenText += script[cursor];
168+
if (cursor + 1 >= script.length) break;
169+
}
158170
}
159171
tokenText = this.processToken(tokenText, tokens, true, TokenTypes.LiteralString);
160172
} else if (symbol != ' ') {

0 commit comments

Comments
 (0)
Please sign in to comment.