Skip to content

Commit 2d570b1

Browse files
mattdeboardmattdeboard
committed
Rewrite of forward/backward token functionality.
The previous incarnation of this is quite slow and very hard to follow. In this commit I am trying to see how concise I can make the forward- and backward-token functions.
1 parent 97b3c64 commit 2d570b1

File tree

4 files changed

+85
-218
lines changed

4 files changed

+85
-218
lines changed

‎elixir-mode-tests.el‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
,@body))
2222

2323
(load "test/elixir-mode-indentation-tests.el")
24-
(load "test/elixir-mode-tokenizer-hl-tests.el")
2524
(load "test/elixir-mode-font-tests.el")
2625

2726
(provide 'elixir-mode-tests)

‎elixir-smie.el‎

Lines changed: 76 additions & 178 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
,regexp)
5454
(pushnew `(,',regex-name . ,(upcase (symbol-name ',name))) elixir-syntax-class-names))))
5555

56-
(elixir-smie-define-regexp-opt op "&&" "||" "!")
56+
;(elixir-smie-define-regexp-opt op "&&" "||" "!")
5757
(elixir-smie-define-regexp dot "\\.")
5858
(elixir-smie-define-regexp comma ",")
5959
(elixir-smie-define-regexp -> "->")
@@ -65,182 +65,81 @@
6565
'(do else catch after rescue -> OP)
6666
"Keywords in which newlines cause confusion for the parser.")
6767

68-
(defun elixir-skip-comment-backward ()
69-
"Skip backwards over all whitespace and comments.
68+
(defvar elixir-smie--operator-regexp
69+
(regexp-opt '("<<<" ">>>" "^^^" "~~~" "&&&" "|||" "===" "!==" "==" "!=" "<="
70+
">=" "<" ">" "&&" "||" "<>" "++" "--" "//"
71+
"/>" "=~" "|>" "->")))
7072

71-
Return non-nil if any line breaks were skipped."
72-
(let ((start-line-no (line-number-at-pos (point))))
73-
(forward-comment (- (point)))
74-
(/= start-line-no (line-number-at-pos (point)))))
75-
76-
(defun elixir-skip-comment-forward ()
77-
"Skip forward over any whitespace and comments.
78-
79-
Return non-nil if any line breaks were skipped."
80-
(let ((start-line-no (line-number-at-pos (point))))
81-
(forward-comment (buffer-size))
82-
(/= start-line-no (line-number-at-pos (point)))))
83-
84-
(defun elixir-smie-next-token-no-lookaround (forwardp)
85-
(block elixir-smie-next-token-no-lookaround
86-
;; First, skip comments; but if any comments / newlines were
87-
;; skipped, the upper level needs to check if they were significant:
88-
(when (if forwardp
89-
(elixir-skip-comment-forward)
90-
(elixir-skip-comment-backward))
91-
(return-from elixir-smie-next-token-no-lookaround "\n"))
92-
(let* ((found-token-class (find-if
93-
(lambda (class-def)
94-
(let ((regex (symbol-value (car class-def))))
95-
(if forwardp
96-
(looking-at regex)
97-
(looking-back regex nil t))))
98-
elixir-syntax-class-names))
99-
(maybe-token
100-
(let ((current-char (if forwardp
101-
(following-char)
102-
(preceding-char))))
103-
(cond ((member current-char
104-
'(?\n ?\;))
105-
(if forwardp
106-
(forward-comment (point-max))
107-
(forward-comment (- (point))))
108-
(string current-char))
109-
(found-token-class
110-
(goto-char (if forwardp
111-
(match-end 0)
112-
(match-beginning 0)))
113-
(if (string= "PARENS" (cdr found-token-class))
114-
(buffer-substring-no-properties (match-beginning 0) (match-end 0))
115-
(cdr found-token-class)))
116-
((when (= ?\" (char-syntax (if forwardp
117-
(following-char)
118-
(preceding-char))))
119-
(if forwardp
120-
(forward-sexp)
121-
(backward-sexp))
122-
"STRING"))))))
123-
(or maybe-token
124-
(downcase
125-
(buffer-substring-no-properties
126-
(point)
127-
(if forwardp
128-
(progn (skip-syntax-forward "'w_")
129-
(point))
130-
(progn (skip-syntax-backward "'w_")
131-
(point)))))))))
132-
133-
(defun elixir-smie-next-token (forwardp)
134-
(block elixir-smie-next-token
135-
(let ((current-token (elixir-smie-next-token-no-lookaround forwardp)))
136-
(when (string= "\n" current-token)
137-
;; This is a newline; if the previous token isn't an OP2, this
138-
;; means the line end marks the end of a statement & we get to
139-
;; scan forward until there's a non-newline token; otherwise,
140-
;; make this line ending something that probably ends the
141-
;; statement (but see below).
142-
(if (save-excursion
143-
(block nil
144-
(let ((token (elixir-smie-next-token-no-lookaround nil)))
145-
(while (and (not (= (point) (point-min)))
146-
(string= "\n" token))
147-
(setq token (elixir-smie-next-token-no-lookaround nil)))
148-
(when (member (intern token) elixir-smie-block-intro-keywords)
149-
(return t)))))
150-
;; it's a continuation line, return the next token after the newline:
151-
(return-from elixir-smie-next-token (elixir-smie-next-token forwardp))
152-
(setq current-token ";")))
73+
(defun elixir-smie--at-dot-call ()
74+
(and (eq ?w (char-syntax (following-char)))
75+
(eq (char-before) ?.)
76+
(not (eq (char-before (1- (point))) ?.))))
15377

154-
;; When reading match statements (the ones with expr -> statements),
155-
;; we need to drop non-; delimiters so the parser knows when a
156-
;; match statement ends and another begins, so scan around point to
157-
;; see if there are any -> within the current block's scope.
158-
159-
;; If the current token is a ";", scan forward to see if the current
160-
;; potential statement contains a "->". If so, scan back to find a
161-
;; "do". If there is a -> there, emit a match-statement-delimiter
162-
;; instead of the ";".
163-
(if (and (string= ";" current-token)
164-
;; Scan ahead:
165-
(let ((level 0)
166-
token)
167-
(save-excursion
168-
(block nil
169-
(while
170-
(and
171-
;; Cursor is not at the end of the buffer...
172-
(not (= (point) (point-max)))
173-
;; ...and the current token is not an empty string...
174-
(not (string= "" token))
175-
;; ...nor a newline nor a semicolon.
176-
(not (or (string= "\n" token) (string= ";" token))))
177-
(setq token (elixir-smie-next-token-no-lookaround t))
178-
;; If we're at the top level and the token is "->",
179-
;; return t
180-
(cond ((and (= level 0) (string= "->" token))
181-
(return t))
182-
;; If token is "do" or "fn", increment level
183-
((find token '("do" "fn") :test 'string=)
184-
(incf level))
185-
;; If token is "end", decrement level
186-
((string= token "end")
187-
(decf level)))))))
188-
;; Scan behind:
189-
(let (token)
190-
(save-excursion
191-
(block nil
192-
(while
193-
(and
194-
;; Cursor is not at the beginning of buffer...
195-
(not (= (point) (point-min)))
196-
;; ...and token is neither empty string, nor "do"/"fn"
197-
(not (string= "" token))
198-
(not (string= "do" token))
199-
(not (string= "fn" token)))
200-
(setq token (elixir-smie-next-token-no-lookaround nil))
201-
(when (string= "->" token)
202-
(return t)))
203-
(when (string= token "do") t)))))
204-
"MATCH-STATEMENT-DELIMITER"
205-
current-token))))
78+
(defun elixir-smie--implicit-semi-p ()
79+
(not (or (memq (char-before) '(?\{ ?\[ ?\,))
80+
(looking-back elixir-smie--operator-regexp (- (point) 3) t))))
20681

20782
(defun elixir-smie-forward-token ()
208-
(elixir-smie-next-token t))
83+
(cond
84+
((and (looking-at "\n") (elixir-smie--implicit-semi-p))
85+
(if (eolp) (forward-char 1) (forward-comment 1))
86+
";")
87+
((looking-at elixir-smie--operator-regexp)
88+
(goto-char (match-end 0))
89+
"OP")
90+
(t (smie-default-forward-token))))
20991

21092
(defun elixir-smie-backward-token ()
211-
(elixir-smie-next-token nil))
93+
(let ((pos (point)))
94+
(forward-comment (- (point)))
95+
(cond
96+
((and (> pos (line-end-position))
97+
(elixir-smie--implicit-semi-p))
98+
";")
99+
((looking-back elixir-smie--operator-regexp (- (point) 3) t)
100+
(goto-char (match-beginning 0))
101+
"OP")
102+
(t (smie-default-backward-token)))))
212103

213104
(defconst elixir-smie-grammar
214105
(smie-prec2->grammar
215-
(smie-bnf->prec2
216-
'((id)
217-
(statements (statement)
218-
(statement ";" statements))
219-
(statement ("def" non-block-expr "do" statements "end")
220-
(non-block-expr "fn" match-statement "end")
221-
(non-block-expr "do" statements "end")
222-
("if" non-block-expr "do" statements "else" statements "end")
223-
("if" non-block-expr "do" statements "end")
224-
("if" non-block-expr "COMMA" "do:" non-block-expr)
225-
("if" non-block-expr "COMMA"
226-
"do:" non-block-expr "COMMA"
227-
"else:" non-block-expr)
228-
("try" "do" statements "after" statements "end")
229-
("try" "do" statements "catch" match-statements "end")
230-
("try" "do" statements "end")
231-
("case" non-block-expr "do" match-statements "end"))
232-
(non-block-expr (non-block-expr "OP" non-block-expr)
233-
(non-block-expr "COMMA" non-block-expr)
234-
("(" statements ")")
235-
("{" statements "}")
236-
("[" statements "]")
237-
("STRING"))
238-
(match-statements (match-statement "MATCH-STATEMENT-DELIMITER" match-statements)
239-
(match-statement))
240-
(match-statement (non-block-expr "->" statements)))
241-
'((assoc "if" "do:" "else:")
242-
(assoc "COMMA")
243-
(left "OP")))))
106+
(smie-merge-prec2s
107+
(smie-bnf->prec2
108+
'((id)
109+
(statements (statement)
110+
(statement ";" statements))
111+
(statement ("def" non-block-expr "do" statements "end")
112+
(non-block-expr "fn" match-statement "end")
113+
(non-block-expr "do" statements "end")
114+
("if" non-block-expr "do" statements "else" statements "end")
115+
("if" non-block-expr "do" statements "end")
116+
("if" non-block-expr "COMMA" "do:" non-block-expr)
117+
("if" non-block-expr "COMMA"
118+
"do:" non-block-expr "COMMA"
119+
"else:" non-block-expr)
120+
("try" "do" statements "after" statements "end")
121+
("try" "do" statements "catch" match-statements "end")
122+
("try" "do" statements "end")
123+
("case" non-block-expr "do" match-statements "end"))
124+
(non-block-expr (non-block-expr "OP" non-block-expr)
125+
(non-block-expr "COMMA" non-block-expr)
126+
("(" non-block-expr ")")
127+
("{" non-block-expr "}")
128+
("[" non-block-expr "]")
129+
("STRING"))
130+
(match-statements (match-statement "MATCH-STATEMENT-DELIMITER" match-statements)
131+
(match-statement))
132+
(match-statement (non-block-expr "->" statements)))
133+
'((assoc "if" "do:" "else:")
134+
(assoc "COMMA")
135+
(left "OP")))
136+
137+
(smie-precs->prec2
138+
'((left "||")
139+
(left "&&")
140+
(nonassoc "=~" "===" "!==" "==" "!=" "<=" ">=" "<" ">")
141+
(left "+" "-" "<<<" ">>>" "^^^" "~~~" "&&&" "|||")
142+
(left "*" "/"))))))
244143

245144
(defvar elixir-smie-indent-basic 2)
246145

@@ -265,26 +164,25 @@ Return non-nil if any line breaks were skipped."
265164
0
266165
elixir-smie-indent-basic))
267166
(`(:after . "OP")
268-
(unless (smie-rule-sibling-p)
269-
elixir-smie-indent-basic))
167+
(cond
168+
((smie-rule-sibling-p) nil)
169+
((smie-rule-hanging-p) (smie-rule-parent elixir-smie-indent-basic))
170+
(t elixir-smie-indent-basic)))
270171
(`(:before . "def") elixir-smie-indent-basic)
271172
;; If the parent token of `->' is `fn', then we want to align to the
272173
;; parent, and offset by `elixir-smie-indent-basic'. Otherwise, indent
273174
;; normally. This helps us work with/indent anonymous function blocks
274175
;; correctly.
275-
(`(:after . "->")
276-
(when (smie-rule-hanging-p)
277-
(if (smie-rule-parent-p "fn")
278-
(smie-rule-parent elixir-smie-indent-basic)
279-
elixir-smie-indent-basic)))
280-
(`(:after . "do")
281-
elixir-smie-indent-basic)
282176
(`(:list-intro . ,(or `"do" `";")) t)
177+
(`(:before . ";")
178+
(cond
179+
((smie-rule-parent-p "after" "catch" "def" "defmodule" "defp" "do" "else"
180+
"fn" "if" "rescue" "try" "unless")
181+
(smie-rule-parent elixir-smie-indent-basic))))
283182
(`(:after . ";")
284183
(if (smie-rule-parent-p "if")
285184
(smie-rule-parent 0)))))
286185

287-
288186
(define-minor-mode elixir-smie-mode
289187
"SMIE-based indentation and syntax for Elixir"
290188
nil nil nil nil

‎test/elixir-mode-indentation-tests.el‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -194,14 +194,18 @@ end
194194

195195
(elixir-def-indentation-test indents-continuation-lines ()
196196
"
197-
has_something(x) &&
198-
has_something(y) ||
199-
has_something(z)
200-
"
201-
"
197+
def foo do
202198
has_something(x) &&
203199
has_something(y) ||
204200
has_something(z)
201+
end
202+
"
203+
"
204+
def foo do
205+
has_something(x) &&
206+
has_something(y) ||
207+
has_something(z)
208+
end
205209
")
206210

207211
(elixir-def-indentation-test indents-continuation-lines-with-comments/1

‎test/elixir-mode-tokenizer-hl-tests.el‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

0 commit comments

Comments
 (0)