debug-ci/main.py at master · rassakra/debug-ci · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#!/usr/bin/env python3
# coding=utf-8

import base64
import json
import spacy
import textacy
import unicodedata
from flask import Flask, request
from flask import Response
from urllib.parse import unquote
from spacy.language import Language

PORT = 8180
app = Flask(__name__)

spacy.prefer_gpu()
nlp = spacy.load("en_core_web_lg")

config = {
    "punct_chars": ['.', '?', '!', ',\n', '*', '- ', ':\n', '۔', '܀',
                    '܁', '܂', '߹',
                    '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄',
                    '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿',
                    '‼', '‽', '⁇', '⁈', '⁉', '⸮', '⸼', '꓿', '꘎', '꘏', '꛳', '꛷', '꡶',
                    '꡷', '꣎', '꣏', '꤯', '꧈', '꧉', '꩝', '꩞', '꩟', '꫰', '꫱', '꯫', '﹒',
                    '﹖', '﹗', '！', '．', '？', '𐩖', '𐩗', '𑁇', '𑁈', '𑂾', '𑂿', '𑃀',
                    '𑃁', '𑅁', '𑅂', '𑅃', '𑇅', '𑇆', '𑇍', '𑇞', '𑇟', '𑈸', '𑈹', '𑈻', '𑈼',
                    '𑊩', '𑑋', '𑑌', '𑗂', '𑗃', '𑗉', '𑗊', '𑗋', '𑗌', '𑗍', '𑗎', '𑗏', '𑗐',
                    '𑗑', '𑗒', '𑗓', '𑗔', '𑗕', '𑗖', '𑗗', '𑙁', '𑙂', '𑜼', '𑜽', '𑜾', '𑩂',
                    '𑩃', '𑪛', '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈',
                    '｡', '。']}

nlp.create_pipe('sentencizer', config=config)

sentencizer = nlp.add_pipe('sentencizer', before="parser")

@Language.component("set_custom_boundaries")
def set_custom_boundaries(doc):
    for token in doc[:-1]:
        if token.text == "\n":
            doc[token.i].is_sent_start = True
    return doc

nlp.add_pipe("set_custom_boundaries", before="parser")


rulerConfig = {
    "validate": True,
    "overwrite_ents": True,
}

entity_ruler = nlp.add_pipe("entity_ruler", config=rulerConfig)
entity_ruler.from_disk("./data/va_train.jsonl")

print(nlp.pipe_names)


@app.route('/')
def search():
    key = unquote(request.args.get('q'))
    res = base64.b64decode(key).decode('utf8', 'replace')
    res = unicodedata.normalize("NFKD", res)

    print(repr(res))

    doc = textacy.make_spacy_doc(res, lang=nlp)

######## DONT CHANGE
    return Response(json.dumps({
        "words": [
            {"text": token.text, "data_type": token.pos_, "dep": token.dep_, "parent": token.head.text,
             "lemma": token.lemma_, "tag": token.tag_}
            for token in doc
        ],
        "entities": [
            {"text": ent.text, "start": ent.start_char, "end": ent.end_char, "label": ent.label_, "id": ent.ent_id_,
             "lemma": ent.lemma_}
            for ent in doc.ents
        ],
        "sentances": [
            {"text": s.text, "start": s.start_char, "end": s.end_char, "label": s.label}
            for s in doc.sents
        ]
    }), mimetype='application/json')


if __name__ == "__main__":
    app.run(host='0.0.0.0')