Skip to content

Commit be455d0

Browse files
mrochfacebook-github-bot
mroch
authored and
facebook-github-bot
committed
[PR] [experimental] flow-parser-bin node extension using native parser
Summary: The js_of_ocaml version of the parser, `flow_parser.js`, is not as efficient as the native ocaml version. OCaml and node can both interface with C/C++ code, so we can expose the parser as a C++ library (see changes in src/parser/) and then consume it from a node extension (see packages/flow-parser-bin/). Closes facebook#5508 Reviewed By: gabelevi Differential Revision: D6552926 Pulled By: mroch fbshipit-source-id: 5b491cb8d88fcbc2517b5e891589bcfe082ac74a
1 parent cda702a commit be455d0

File tree

13 files changed

+339
-0
lines changed

13 files changed

+339
-0
lines changed

‎_tags

+1
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ true: package(dtoa), package(wtf8)
77
<src/parser/*.ml*>: warn(-39)
88
<**/node_modules/**>: -traverse
99
<_obuild>: -traverse
10+
<packages>: -traverse

‎packages/flow-parser-bin/.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
build/
2+
include/
3+
lib/
4+
node_modules/

‎packages/flow-parser-bin/Makefile

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright (c) 2017-present, Facebook, Inc.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
OCAML_HEADERS=\
7+
include/caml/alloc.h \
8+
include/caml/callback.h \
9+
include/caml/config.h \
10+
include/caml/memory.h \
11+
include/caml/misc.h \
12+
include/caml/mlvalues.h
13+
14+
OCAML_PATH=$(shell ocamlc -where)
15+
16+
.PHONY: lib/libflowparser.a
17+
lib/libflowparser.a:
18+
@mkdir -p lib
19+
$(MAKE) -C ../../src/parser lib/libflowparser.a
20+
cp ../../src/parser/lib/libflowparser.a "$@"
21+
22+
include/flowparser/libflowparser.h: ../../src/parser/libflowparser.h
23+
@mkdir -p "$(dir $@)"
24+
cp "$<" "$@"
25+
26+
$(OCAML_HEADERS): include/%: $(OCAML_PATH)/%
27+
@mkdir -p "$(dir $@)"
28+
cp "$<" "$@"
29+
30+
.PHONY: deps
31+
deps: lib/libflowparser.a include/flowparser/libflowparser.h $(OCAML_HEADERS)
32+
33+
.PHONY: clean
34+
clean:
35+
rm -rf include/ lib/

‎packages/flow-parser-bin/binding.gyp

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"targets": [
3+
{
4+
"target_name": "flow_parser",
5+
"sources": [
6+
"src/flow_parser_node.cc",
7+
"../lib/libflowparser.a",
8+
],
9+
"include_dirs": [
10+
"include/",
11+
"<!(node -e \"require('nan')\")",
12+
],
13+
"libraries": [
14+
"-L../lib",
15+
"-lflowparser",
16+
],
17+
"conditions": [
18+
['OS=="mac"', {
19+
"xcode_settings": {
20+
"OTHER_LDFLAGS": [
21+
# suppress this warning:
22+
# ld: warning: could not create compact unwind for
23+
# _caml_start_program: dwarf uses DW_CFA_same_value
24+
"-Wl,-no_compact_unwind",
25+
],
26+
},
27+
}],
28+
],
29+
}
30+
]
31+
}

‎packages/flow-parser-bin/index.js

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/**
2+
* Copyright (c) 2017-present, Facebook, Inc.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*/
7+
8+
var native = require('bindings')('flow_parser.node');
9+
exports.parse = native.parse;

‎packages/flow-parser-bin/package.json

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "flow-parser-bin",
3+
"version": "0.61.0",
4+
"description": "The Flow JavaScript parser, via bindings to the native OCaml implementation",
5+
"main": "index.js",
6+
"repository": "https://github.com/facebook/flow.git",
7+
"author": "Marshall Roch <[email protected]>",
8+
"license": "MIT",
9+
"dependencies": {
10+
"bindings": "^1.3.0",
11+
"nan": "^2.8.0"
12+
}
13+
}

‎packages/flow-parser-bin/src/flow_parser_node.cc

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/**
2+
* Copyright (c) 2013-present, Facebook, Inc.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*/
7+
8+
#include <nan.h>
9+
#include <flowparser/libflowparser.h>
10+
11+
using namespace std;
12+
using namespace v8;
13+
using namespace flowparser;
14+
15+
class V8Translator : public AbstractTranslator<Local<Value>> {
16+
public:
17+
Local<Value> convert_string(char *str) {
18+
return Nan::New(str).ToLocalChecked();
19+
};
20+
Local<Value> convert_number(double n) {
21+
return Nan::New(n);
22+
}
23+
Local<Value> convert_bool(long b) {
24+
return Nan::New<Boolean>(b);
25+
}
26+
Local<Value> convert_null() {
27+
return Nan::Null();
28+
}
29+
Local<Value> convert_undefined() {
30+
return Nan::Undefined();
31+
}
32+
Local<Value> convert_object(value props) {
33+
Local<Object> obj = Nan::New<Object>();
34+
value prop;
35+
36+
while (list_has_next(props)) {
37+
prop = list_head(props);
38+
Nan::Set(obj,
39+
convert_string(get_prop_key(prop)),
40+
convert_json(get_prop_value(prop))
41+
);
42+
props = list_tail(props);
43+
}
44+
45+
return obj;
46+
}
47+
48+
Local<Value> convert_array(value items) {
49+
Local<Array> arr = Nan::New<Array>();
50+
value head;
51+
size_t i = 0;
52+
while (list_has_next(items)) {
53+
head = list_head(items);
54+
arr->Set(i, convert_json(head));
55+
items = list_tail(items);
56+
i++;
57+
}
58+
return arr;
59+
}
60+
};
61+
62+
NAN_METHOD(parse) {
63+
Nan::HandleScope scope;
64+
V8Translator converter;
65+
String::Utf8Value arg(info[0]);
66+
string content = string(*arg);
67+
Local<Value> result = converter.parse(content.c_str());
68+
info.GetReturnValue().Set(result);
69+
}
70+
71+
NAN_MODULE_INIT(Init) {
72+
flowparser::init();
73+
NAN_EXPORT(target, parse);
74+
}
75+
76+
NODE_MODULE(flow_parser, Init)

‎packages/flow-parser-bin/yarn.lock

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
2+
# yarn lockfile v1
3+
4+
5+
bindings@^1.3.0:
6+
version "1.3.0"
7+
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7"
8+
9+
nan@^2.8.0:
10+
version "2.8.0"
11+
resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a"

‎src/parser/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
lib/
12
lex_test
23
parse_test
34
lexer_flow.ml

‎src/parser/Makefile

+7
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ build-parser:
4444
cd $(TOP); \
4545
$(OCB) -no-links $(REL_DIR)/parser_flow.cmxa
4646

47+
.PHONY: lib/libflowparser.a
48+
lib/libflowparser.a:
49+
@mkdir -p lib
50+
cp "$(shell ocamlc -where)"/libasmrun.a "$@"
51+
cd $(TOP) && $(OCB) -no-links $(REL_DIR)/libflowparser.native.o
52+
ar rcs "$@" "$(TOP)/_build/$(REL_DIR)/libflowparser.native.o"
53+
4754
js:
4855
cd $(TOP); \
4956
$(OCB) -pkgs js_of_ocaml $(REL_DIR)/flow_parser_dot_js.byte; \

‎src/parser/_tags

+1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
true: package(sedlex), package(wtf8)
22
<lexer.ml>: warn(-39)
3+
<lib>: -traverse

‎src/parser/libflowparser.h

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/**
2+
* Copyright (c) 2013-present, Facebook, Inc.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*/
7+
8+
#define CAML_NAME_SPACE
9+
10+
#include <caml/alloc.h>
11+
#include <caml/mlvalues.h>
12+
#include <caml/memory.h>
13+
#include <caml/callback.h>
14+
15+
namespace flowparser {
16+
17+
static value list_head(value props) {
18+
return Field(props, 0);
19+
}
20+
21+
static bool list_has_next(value props) {
22+
return props != Val_emptylist;
23+
}
24+
25+
static value list_tail(value props) {
26+
return Field(props, 1);
27+
}
28+
29+
static char* get_prop_key(value prop) {
30+
return String_val(Field(prop, 0));
31+
}
32+
33+
static value get_prop_value(value prop) {
34+
return Field(prop, 1);
35+
}
36+
37+
template <class T>
38+
class AbstractTranslator {
39+
public:
40+
virtual T convert_string(char *str) = 0;
41+
virtual T convert_number(double n) = 0;
42+
virtual T convert_bool(long b) = 0;
43+
virtual T convert_null() = 0;
44+
virtual T convert_undefined() = 0;
45+
virtual T convert_object(value props) = 0;
46+
virtual T convert_array(value items) = 0;
47+
48+
T convert_json(value v) {
49+
if (Is_long(v)) {
50+
if (Long_val(v) == 0) {
51+
return convert_null();
52+
}
53+
// no other immediate values should exist
54+
return convert_undefined(); // TODO: raise v8 exception
55+
};
56+
switch (Tag_val(v)) {
57+
// JObject
58+
case 0:
59+
return convert_object(Field(v, 0));
60+
61+
// JArray
62+
case 1:
63+
return convert_array(Field(v, 0));
64+
65+
// JString
66+
case 2:
67+
return convert_string(String_val(Field(v, 0)));
68+
69+
// JNumber
70+
case 3:
71+
return convert_number(Double_val(Field(v, 0)));
72+
73+
// JBool
74+
case 4:
75+
return convert_bool(Long_val(Field(v, 0)));
76+
77+
default:
78+
// no other tags exist!
79+
return convert_undefined(); // TODO: raise v8 exception
80+
}
81+
}
82+
83+
T parse(const char *content) {
84+
CAMLparam0();
85+
CAMLlocal2(content_val, result_val);
86+
87+
static value * func = NULL;
88+
if (func == NULL) {
89+
func = caml_named_value("flow_parse");
90+
}
91+
92+
content_val = caml_copy_string(content);
93+
result_val = caml_callback(*func, content_val);
94+
95+
CAMLreturnT(T, convert_object(Field(result_val, 0)));
96+
}
97+
};
98+
99+
void init() {
100+
char *argv[] = {NULL};
101+
caml_startup(argv);
102+
}
103+
104+
} // namespace flow

‎src/parser/libflowparser.ml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
(**
2+
* Copyright (c) 2013-present, Facebook, Inc.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*)
7+
8+
(* do NOT change the order of these constructors! the C code relies on ocaml's
9+
memory representation, which is order dependent. *)
10+
type json =
11+
| JObject of (string * json) list
12+
| JArray of json list
13+
| JString of string
14+
| JNumber of float
15+
| JBool of bool
16+
| JNull
17+
18+
module AbstractTranslator : (
19+
Estree_translator.Translator with type t = json
20+
) = struct
21+
type t = json
22+
let string x = JString x
23+
let bool x = JBool x
24+
let obj props = JObject props
25+
let array arr = JArray arr
26+
let number x = JNumber x
27+
let null = JNull
28+
let regexp _loc _pattern _flags = JNull
29+
end
30+
31+
module Translate = Estree_translator.Translate (AbstractTranslator) (struct
32+
(* TODO: make these configurable via CLI flags *)
33+
let include_comments = true
34+
let include_locs = true
35+
end)
36+
37+
(* TODO: parse_options *)
38+
let parse content =
39+
let (ast, errors) = Parser_flow.program ~fail:false content in
40+
match Translate.program ast with
41+
| JObject params ->
42+
JObject (("errors", Translate.errors errors)::params)
43+
| _ -> assert false
44+
45+
(* TODO: register/handle Parse_error.Error *)
46+
let () = Callback.register "flow_parse" parse

0 commit comments

Comments
 (0)