Skip to content

Commit d717bf5

Browse files
RReverserinikulin
RReverser
authored and
inikulin
committed
Generate feedback tests (inikulin#138)
* Simplify testing of parser feedback; fix chunked processing. Adds task for generation of parser feedback tests in HTML5Lib tokenization format, so that they can be readable and reusable instead of generating them in runtime. Since it's now in the same format as regular tokenization tests, merged parser_feedback_test logic into tokenizer_test (this required only extra parameter to be passed around the runner). Additionally this helped to test parser feedback with chunked input and identify & fix an error related to extraneous newlines after hibernation. * Add pregenerated parser feedback tests. Added them in a separate commit for cleaner diffs and easier review.
1 parent d247287 commit d717bf5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+41978
-230
lines changed

‎Gulpfile.js

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,81 @@ gulp.task('lint', function () {
110110
.pipe(eslint.failAfterError());
111111
});
112112

113+
gulp.task('update-feedback-tests', function () {
114+
var Parser = require('./lib/Parser');
115+
var Tokenizer = require('./lib/tokenizer');
116+
var defaultTreeAdapter = require('./lib/tree_adapters/default');
117+
var testUtils = require('./test/test_utils');
118+
119+
function appendToken(dest, token) {
120+
switch (token.type) {
121+
case Tokenizer.EOF_TOKEN:
122+
return false;
123+
case Tokenizer.NULL_CHARACTER_TOKEN:
124+
case Tokenizer.WHITESPACE_CHARACTER_TOKEN:
125+
token.type = Tokenizer.CHARACTER_TOKEN;
126+
/* falls through */
127+
case Tokenizer.CHARACTER_TOKEN:
128+
if (dest.length > 0 && dest[dest.length - 1].type === Tokenizer.CHARACTER_TOKEN) {
129+
dest[dest.length - 1].chars += token.chars;
130+
return true;
131+
}
132+
break;
133+
}
134+
dest.push(token);
135+
return true;
136+
}
137+
138+
function collectParserTokens(html) {
139+
var tokens = [];
140+
var parser = new Parser();
141+
142+
parser._processInputToken = function (token) {
143+
Parser.prototype._processInputToken.call(this, token);
144+
145+
// Needed to split attributes of duplicate <html> and <body>
146+
// which are otherwise merged as per tree constructor spec
147+
if (token.type === Tokenizer.START_TAG_TOKEN)
148+
token.attrs = token.attrs.slice();
149+
150+
appendToken(tokens, token);
151+
};
152+
153+
parser.parse(html);
154+
155+
return tokens.map(testUtils.convertTokenToHtml5Lib);
156+
}
157+
158+
return gulp
159+
.src(['test/data/tree_construction/*.dat', 'test/data/tree_construction_regression/*.dat'])
160+
.pipe(through.obj(function (file, encoding, callback) {
161+
var tests = testUtils.parseTreeConstructionTestData(
162+
file.contents.toString(),
163+
defaultTreeAdapter
164+
);
165+
166+
var out = {
167+
tests: tests.filter(function (test) {
168+
return !test.fragmentContext; // TODO
169+
}).map(function (test) {
170+
var input = test.input;
171+
172+
return {
173+
description: testUtils.addSlashes(input),
174+
input: input,
175+
output: collectParserTokens(input)
176+
};
177+
})
178+
};
179+
180+
file.contents = new Buffer(JSON.stringify(out, null, 4));
181+
182+
callback(null, file);
183+
}))
184+
.pipe(rename({ extname: '.test' }))
185+
.pipe(gulp.dest('test/data/parser_feedback'));
186+
});
187+
113188
gulp.task('test', ['lint'], function () {
114189
return gulp
115190
.src('test/fixtures/*_test.js')

‎lib/sax/parser_feedback_simulator.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ ParserFeedbackSimulator.prototype.getNextToken = function () {
3636
}
3737

3838
else if (this.skipNextNewLine) {
39-
this.skipNextNewLine = false;
39+
if (token.type !== Tokenizer.HIBERNATION_TOKEN)
40+
this.skipNextNewLine = false;
4041

4142
if (token.type === Tokenizer.WHITESPACE_CHARACTER_TOKEN && token.chars[0] === '\n') {
4243
if (token.chars.length === 1)

0 commit comments

Comments
 (0)