Skip to content

Commit 8988990

Browse files
jacobbogersjacobbogers
committed
using regexp to scan strings and wrapped it into a "processor" type
1 parent 9fbcfb5 commit 8988990

File tree

6 files changed

+182
-76
lines changed

6 files changed

+182
-76
lines changed

‎src/channel.ts‎

Lines changed: 153 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,46 @@ import * as debug from 'debug'
22

33
import { ISimpleToken, IRangeToken } from './IToken';
44
import { IModule, IModuleEnums } from './module'
5-
import { IMatcher, IMatcherState } from './matchers'
5+
import { IMatcher, IMatcherState, createClassMatcher } from './matchers'
66
import { ITokenEmitter } from './tokenProducers'
77
import { isComment, isContinue, binarySearch, last } from './helpers'
8+
import { ws } from './classes'
9+
import { createTokenEmitter, rangeProducer } from './tokenProducers'
810

911
const printer = debug('IChannel')
1012

13+
export interface linePos {
14+
ln: number;
15+
col: number;
16+
}
17+
export interface Snippet {
18+
line: string;
19+
f: number;
20+
t: number;
21+
}
22+
23+
export interface Processed {
24+
snippets: Snippet[];
25+
tokens: IRangeToken[];
26+
}
27+
28+
function compare(a: any, b: any): 0 | -1 | 1 {
29+
const df = a.f - b.f
30+
if (df > 0) return 1
31+
if (df < 0) return -1
32+
if (a.t !== undefined && b.t !== undefined) {
33+
const dt = a.t - b.t
34+
if (dt > 0) return 1
35+
if (dt < 0) return -1
36+
}
37+
return 0
38+
}
39+
40+
const search = binarySearch(compare)
41+
const wsMatcher = createClassMatcher(ws, '>1')
42+
const wsEmitter = createTokenEmitter(rangeProducer, wsMatcher)
43+
44+
1145
export interface IChannel<T extends ISimpleToken> {
1246
mod: IModule;
1347
name: string;
@@ -57,106 +91,185 @@ export function createLogicalEOLChannel<T extends ISimpleToken>(ch: IChannel<T>)
5791
if (ch !== ch.mod.channels.get('lf')) {
5892
throw new TypeError(`source "lf" channel is not registered with a module`)
5993
}
94+
let tokens: T[] = []
6095
const vCh: IChannel<T> = {
6196
mod: ch.mod,
62-
tokens: [], //vtokens
97+
tokens,
6398
name: 'vlf',
6499
process() {
65-
const tokens = this.tokens = []
100+
tokens = []
101+
const lftok = ch.tokens.slice(0)
102+
const raw = ch.mod.raw
66103
let prev = 0
67-
for (let i = 0; i < ch.tokens.length; i++) {
104+
for (let i = 0; i < lftok.length; i++) {
68105
const pos = ch.tokens[i].f
69-
const line = ch.mod.raw.slice(prev, pos)
106+
const line = raw.slice(prev, pos)
70107
prev = pos + 1
71108
if (isContinue(line)) {
72-
if (tokens.length === 0) {
109+
if (i === 0) {
73110
const err = `first line cannot be continuation: [${line}]`
74111
printer(err)
75112
throw new Error(err)
76113
}
77-
tokens[tokens.length - 1] = ch.tokens[i]
114+
// update in place
115+
tokens[tokens.length - 1] = lftok[i]
78116
continue
79117
}
80-
tokens.push(ch.tokens[i])
118+
tokens.push(lftok[i])
81119
}
82120
}
83121
}
84122
ch.mod.channels.set(vCh.name, vCh)
85123
return vCh
86124
}
87125

88-
export function createCommentsChannel<T extends ISimpleToken>(ch: IChannel<T>): IChannel<T> {
126+
export function createCommentsChannel(ch: IChannel<ISimpleToken>): IChannel<IRangeToken> {
89127

90128
const vlf = ch.mod.channels.get('vlf')
91129
const lf = ch.mod.channels.get('lf')
92-
130+
93131
if (ch !== vlf && ch !== lf) {
94132
throw new TypeError(`source "lf/vlf" channel is not registered with a module`)
95133
}
96-
97-
const comm: IChannel<T> = {
134+
const _lf = vlf || lf
135+
const tokens: IRangeToken[] = []
136+
const raw = _lf.mod.raw
137+
const comm: IChannel<IRangeToken> = {
98138
mod: ch.mod,
99-
tokens: [], //vtokens
139+
tokens,
100140
name: 'comments',
101141
process() {
102-
const tokens = this.tokens = []
103-
const lftok = ch.tokens.slice(0) //copy
104-
142+
tokens.splice(0)
143+
const lftok = _lf.tokens.slice(0) //copy
105144
let prev = 0
106-
const raw = ch.mod.raw
107145
for (let i = 0; i < lftok.length; i++) {
108146
const pos = lftok[i].f
109147
const line = raw.slice(prev, pos)
110-
111148
if (isComment(line)) {
112-
tokens.push({ f: prev, t: pos })
113-
149+
tokens.push({ f: prev, t: pos - 1 })
114150
}
115151
prev = pos + 1
116152
}
117153
const lastf = last(lftok).f
118-
if (lastf < raw.length-1){
119-
const line = raw.slice(lastf)
120-
if (isComment(line)){
121-
tokens.push({f:lastf+1,t:raw.length-1})
122-
}
154+
if (lastf < raw.length - 1) {
155+
const line = raw.slice(lastf + 1)
156+
if (isComment(line)) {
157+
tokens.push({ f: lastf + 1, t: raw.length - 1 })
158+
}
123159
}
124160
}
125161
}
126162
ch.mod.channels.set(comm.name, comm)
127163
return comm
128164
}
129165

130-
export function createSourceChannel<T extends ISimpleToken>(ch: IChannel<T>): IChannel<T> {
166+
export function createSourceChannel(ch: IChannel<ISimpleToken>): IChannel<IRangeToken> {
131167

132168
const vlf = ch.mod.channels.get('vlf')
133-
const comms = ch.mod.channels.get('comments')
134-
if (vlf !== ch){
169+
const comms = ch.mod.channels.get('comments') as IChannel<IRangeToken>
170+
if (vlf !== ch) {
135171
throw new TypeError(`source "vlf" channel is not registered with a module`)
136172
}
137-
if (comms === undefined){
173+
if (comms === undefined) {
138174
throw new TypeError(`source "comments" channel is not registered with a module`)
139175
}
140-
const source: IChannel<T> = {
176+
const tokens: IRangeToken[] = []
177+
178+
const source: IChannel<IRangeToken> = {
141179
mod: ch.mod,
142-
tokens: [], //vtokens
180+
tokens, //vtokens
143181
name: 'source',
144182
process() {
145-
const tokens = this.tokens = []
183+
tokens.splice(0) // delete in palce
184+
const lftok = vlf.tokens.slice(0) //copy
185+
const raw = vlf.mod.raw
146186
let prev = 0
147-
for (let i = 0; i < ch.tokens.length; i++) {
148-
const pos = ch.tokens[i].f
149-
const line = ch.mod.raw.slice(prev, pos)
150-
151-
if (isComment(line)) {
152-
tokens.push({ f: prev, t: pos })
153-
187+
const lastf = last(lftok).f
188+
for (let i = 0; i < lftok.length; i++) {
189+
const pos = lftok[i].f
190+
const line = raw.slice(prev, pos)
191+
if (!isComment(line)) {
192+
tokens.push({ f: prev, t: pos - 1 })
154193
}
155194
prev = pos + 1
195+
}
196+
if (lastf < raw.length - 1) {
197+
const line = raw.slice(lastf + 1)
198+
if (!isComment(line)) {
199+
tokens.push({ f: lastf + 1, t: raw.length - 1 })
200+
}
201+
}
202+
}
203+
}
204+
ch.mod.channels.set(source.name, source)
205+
return source
206+
}
156207

208+
export function createWSChannel(ch: IChannel<IRangeToken>): IChannel<IRangeToken> {
209+
210+
const vlf = ch.mod.channels.get('vlf') as IChannel<ISimpleToken>
211+
const source = ch.mod.channels.get('source') as IChannel<IRangeToken>
212+
if (vlf !== ch) {
213+
throw new TypeError(`source "vlf" channel is not registered with a module`)
214+
}
215+
if (source === undefined) {
216+
throw new TypeError(`source "comments" channel is not registered with a module`)
217+
}
218+
const raw = vlf.mod.raw
219+
const tokens: IRangeToken[] = []
220+
const nonWSSource: Snippet[] = []
221+
const ws: IChannel<IRangeToken> = {
222+
mod: ch.mod,
223+
tokens: [], //vtokens
224+
name: 'ws',
225+
process() {
226+
tokens.splice(0)
227+
nonWSSource.splice(0)
228+
const srctok = source.tokens.slice(0) //copy
229+
for (let i = 0; i < srctok.length; i++) {
230+
const { f, t } = srctok[i]
231+
let snip = { line: raw.slice(f, t + 1), f, t }
232+
// split out continueation lines
233+
const { snippets, tokens: _tokens } = processLineContinuation(snip)
234+
tokens.splice(0, 0, ..._tokens)
235+
snippets.map(processWS).forEach(({ snippets: snips, tokens: toks }) => {
236+
tokens.splice(0, 0, ...toks)
237+
nonWSSource.splice(0, 0, ...snips)
238+
})
239+
// here the ws token need to be extracted from line
157240
}
241+
//sort ws tokens because there will be continue line stuff here!!
158242
}
159243
}
160244
ch.mod.channels.set(source.name, source)
161245
return source
162-
}
246+
}
247+
248+
export function createProcessor(regex: RegExp) {
249+
250+
return function process(s: Snippet): Processed {
251+
const { line, f, t } = s
252+
const found = line.match(regex);
253+
const rc = {
254+
snippets: [s],
255+
tokens: []
256+
}
257+
258+
if (found) {
259+
const first = line.slice(0, found.index)
260+
const second = line.slice(found.index + found[0].length)
261+
rc.snippets[0] = { line: first, f, t: f + first.length - 1 }
262+
rc.tokens[0] = { f: f + found.index, t: f + found.index + found[0].length - 1 }
263+
if (second) {
264+
const rv = process({ line: second, f: f + found.index + found[0].length, t })
265+
rc.tokens.splice(0, 0, ...rv.tokens)
266+
rc.snippets.splice(0, 0, ...rv.snippets)
267+
}
268+
}
269+
return rc
270+
}
271+
}
272+
273+
export const processLineContinuation = createProcessor(/\n\s{5}[^\s]/)
274+
export const processWS = createProcessor(/[\s\t]+/)
275+

‎src/helpers.ts‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -45,22 +45,6 @@ export function mergeSort(sort = (a, b) => a > b ? 1 : a < b ? -1 : 0) {
4545
}
4646
}
4747

48-
/*
49-
function binary_search(A, n, T):
50-
L := 0
51-
R := n − 1
52-
while L <= R:
53-
m := floor((L + R) / 2)
54-
if A[m] < T:
55-
L := m + 1
56-
else if A[m] > T:
57-
R := m - 1
58-
else:
59-
return m
60-
return unsuccessfu
61-
// compare the arrays item by item and return the concatenated result
62-
*/
63-
6448
export function binarySearch<T>(compare: (a: T, b: T) => (0 | 1 | -1)) {
6549

6650
return function useArr(list: T[], value: T) {
@@ -95,11 +79,3 @@ export function binarySearch<T>(compare: (a: T, b: T) => (0 | 1 | -1)) {
9579
export function last<T>(arr: T[]): T {
9680
return arr[arr.length-1]
9781
}
98-
99-
//const list = [2, 5, 8, 9, 13, 45, 67, 99]
100-
///console.log(binarySearch(list, 99)) // 7 -> returns the index of the item
101-
102-
103-
104-
//const list = [2, 5, 1, 3, 7, 2, 3, 8, 6, 3]
105-
//console.log(mergeSort(list)) // [ 1, 2, 2, 3, 3, 3, 5, 6, 7, 8 ]

‎src/index.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,16 @@ import { createModule } from './module'
22
import { createClassMatcher, createLiteralMatcher } from './matchers'
33
import { simpleProducer, rangeProducer, createTokenEmitter } from './tokenProducers'
44
import { lf, ws } from './classes'
5-
import { createChannel, createLogicalEOLChannel, createCommentsChannel } from './channel'
5+
import {
6+
createChannel,
7+
createLogicalEOLChannel,
8+
createCommentsChannel,
9+
createProcessor,
10+
createWSChannel,
11+
createSourceChannel,
12+
13+
14+
} from './channel'
615
import { aLoad } from './fsutils'
716
import { mergeSort, binarySearch } from './helpers'
817

@@ -19,7 +28,10 @@ export {
1928
createLogicalEOLChannel,
2029
createCommentsChannel,
2130
mergeSort,
22-
binarySearch
31+
binarySearch,
32+
createProcessor,
33+
createWSChannel,
34+
createSourceChannel
2335
}
2436

2537
export default {
@@ -35,6 +47,9 @@ export default {
3547
createLogicalEOLChannel,
3648
createCommentsChannel,
3749
mergeSort,
38-
binarySearch
50+
binarySearch,
51+
createProcessor,
52+
createWSChannel,
53+
createSourceChannel
3954
}
4055

‎src/matchers.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ export const createClassMatcher = (_class, pattern: string | number = 1) => {
3131
return { oc, ob, oe, os }
3232
}
3333

34-
return function check(c, i): Partial<IMatcherState> {
34+
return function check(c, i): IMatcherState {
3535
// if not found
3636
// -> prev state 'em'?
3737
// -> emit matched value
@@ -108,7 +108,7 @@ export const createClassMatcher = (_class, pattern: string | number = 1) => {
108108
const { oc, ob, oe } = setState('em', _, _, _)
109109
return { s: 'em', b: ob, e: oe, count: oc }
110110
}
111-
return { s: 'te', count }
111+
return { s: 'te', count } as any
112112
}
113113
if (count === pattern) {
114114
const { oc, ob, oe } = setState('in', i, i, 0)

‎src/tokenProducers.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { ISimpleToken, IRangeToken } from './IToken';
66
export type IProducer<T extends ISimpleToken> = (m: IMatcherState) => T
77
export type ITokenEmitter<T extends ISimpleToken> = (c: string, i: number) => (T | undefined)
88

9-
function createProducer<T extends ISimpleToken>(check: IProducer<T>): IProducer<any> {
9+
function createProducer<T extends ISimpleToken>(check: IProducer<T>): IProducer<T> {
1010

1111
return function (am: IMatcherState) {
1212
const rc = check(am)
@@ -31,11 +31,12 @@ export const rangeProducer = createProducer(m=>{
3131
return {
3232
f: m.b,
3333
t: m.e
34-
}
34+
} as IRangeToken
3535
}
3636
})
3737

38-
export function createTokenEmitter<T extends ISimpleToken>(tp: IProducer<T>, matcher: IMatcher) {
38+
export function createTokenEmitter<T extends ISimpleToken>
39+
(tp: IProducer<T>, matcher: IMatcher): (c:string, i:number) => T| undefined {
3940
return function emitToken(c: string, i: number): T | undefined {
4041
const ms = matcher(c, i)
4142
if (ms && ms.s === 'm') {

0 commit comments

Comments
 (0)