From ed10a239606973f34bdf3e2942fa385387ceb4ea Mon Sep 17 00:00:00 2001 From: Wenmin Wu Date: Tue, 17 Sep 2019 11:20:21 +0800 Subject: [PATCH] add jupyter_tabnine extension --- README.md | 11 + src/jupyter_contrib_nbextensions/__init__.py | 20 +- .../nbextensions/jupyter_tabnine/README.md | 34 ++ .../nbextensions/jupyter_tabnine/main.css | 41 ++ .../nbextensions/jupyter_tabnine/main.js | 556 ++++++++++++++++++ .../nbextensions/jupyter_tabnine/tabnine.yaml | 47 ++ src/jupyter_tabnine/__init__.py | 0 src/jupyter_tabnine/handler.py | 14 + src/jupyter_tabnine/tabnine.py | 190 ++++++ 9 files changed, 912 insertions(+), 1 deletion(-) create mode 100644 src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/README.md create mode 100644 src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.css create mode 100644 src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.js create mode 100644 src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/tabnine.yaml create mode 100644 src/jupyter_tabnine/__init__.py create mode 100644 src/jupyter_tabnine/handler.py create mode 100644 src/jupyter_tabnine/tabnine.py diff --git a/README.md b/README.md index 3a33cbc18..a2b3a5847 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,17 @@ To disable the extension again, use jupyter nbextension disable +**Notice**, if you are using the notebook extension which need request the server extension. +You need to enable the server extension before starting the jupyter notebook. +This operation only needs to be done once. Currently, there is only one notebook extension +needs the server extension. That is `jupyter_tabnine`. To enable it's server extension, use + + jupyter serverextension enable --py jupyter_tabnine + +To disable it's server extension again, use + + jupyter serverextension disable --py jupyter_tabnine + **Alternatively**, and more conveniently, you can use the [jupyter_nbextensions_configurator](https://github.com/Jupyter-contrib/jupyter_nbextensions_configurator) server extension, which is installed as a dependency of this repo, and can be diff --git a/src/jupyter_contrib_nbextensions/__init__.py b/src/jupyter_contrib_nbextensions/__init__.py index c03a6c7c0..f086aa2da 100644 --- a/src/jupyter_contrib_nbextensions/__init__.py +++ b/src/jupyter_contrib_nbextensions/__init__.py @@ -3,13 +3,18 @@ import os import jupyter_nbextensions_configurator +from notebook.utils import url_path_join as ujoin +from jupyter_tabnine.handler import TabNineHandler +from jupyter_tabnine.tabnine import TabNine __version__ = '0.5.1' def _jupyter_server_extension_paths(): """Magically-named function for jupyter extension installations.""" - return [] + return [{ + 'module': 'jupyter_tabnine', + }] def _jupyter_nbextension_paths(): @@ -32,3 +37,16 @@ def _jupyter_nbextension_paths(): # _also_ in the `nbextension/` namespace require=nbext['require'], ) for nbext in specs] + + +def load_jupyter_server_extension(nb_server_app): + """ + Called when the extension is loaded. + Args: + nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance. + """ + web_app = nb_server_app.web_app + host_pattern = '.*$' + route_pattern = ujoin(web_app.settings['base_url'], '/tabnine') + tabnine = TabNine() + web_app.add_handlers(host_pattern, [(route_pattern, TabNineHandler, {'tabnine': tabnine})]) \ No newline at end of file diff --git a/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/README.md b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/README.md new file mode 100644 index 000000000..a1d5822f2 --- /dev/null +++ b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/README.md @@ -0,0 +1,34 @@ +Jupyter TabNine +========== +This extension provides code auto-completion based on deep learning. + +* Author: Wenmin Wu +* Repository: https://github.com/wenmin-wu/jupyter-tabnine +* Email: wuwenmin1991@gmail.com + +Options +------- + +* `jupyter_tabnine.before_line_limit`: + maximum number of lines before for context generation, + too many lines will slow down the request. -1 means Infinity, + thus the lines will equal to number of lines before current line. + +* `jupyter_tabnine.after_line_limit`: + maximum number of lines after for context generation, + too many lines will slow down the request. -1 means Infinity, + thus the lines will equal to number of lines after current line. + +* `jupyter_tabnine.options_limit`: + maximum number of options that will be shown + +* `jupyter_tabnine.assist_active`: + Enable continuous code auto-completion when notebook is first opened, or + if false, only when selected from extensions menu. + +* `jupytertabnine.assist_delay`: + delay in milliseconds between keypress & completion request. + +* `jupyter_tabnine.remote_server_url`: + remote server url, you may want to use a remote server to handle client request. + This can spped up the request handling depending on the server configuration. Refer to [repo doc](https://github.com/wenmin-wu/jupyter-tabnine) to see how to deploy remote server. \ No newline at end of file diff --git a/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.css b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.css new file mode 100644 index 000000000..e7b8c1ad4 --- /dev/null +++ b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.css @@ -0,0 +1,41 @@ +.complete-container { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + vertical-align: middle; + font-family: monospace, monospace; +} + +.complete-block { + min-width: 200px; + margin: 2px; + padding: 1px; + display: inline-block; +} + +.user-message { + margin: 2px; + padding: 1px; + display: list-item; +} + +.user-message span { + color: #CD5C5C; +} + + + +.complete-word { + width: auto; + text-align: left; +} + +.complete-detail { + text-align: right; +} + + +.complete-dropdown-content { + background-color: rgb(255, 255, 255, 0.85); + box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.5); +} diff --git a/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.js b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.js new file mode 100644 index 000000000..b65e4704b --- /dev/null +++ b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/main.js @@ -0,0 +1,556 @@ +define([ + 'base/js/namespace', + 'base/js/keyboard', + 'base/js/utils', + 'jquery', + 'module', + 'notebook/js/cell', + 'notebook/js/codecell', + 'notebook/js/completer', + 'require' +], function ( + Jupyter, + keyboard, + utils, + $, + module, + cell, + codecell, + completer, + requirejs +) { + 'use strict'; + + var assistActive; + + var config = { + assist_active: true, + options_limit: 10, + assist_delay: 0, + before_line_limit: -1, + after_line_limit: -1, + remote_server_url: '', + } + + var logPrefix = '[' + module.id + ']'; + var baseUrl = utils.get_body_data('baseUrl'); + var requestInfo = { + "version": "1.0.7", + "request": { + "Autocomplete": { + "filename": Jupyter.notebook.notebook_path.replace('.ipynb', '.py'), + "before": "", + "after": "", + "region_includes_beginning": false, + "region_includes_end": false, + "max_num_results": config.options_limit, + } + } + } + + var Cell = cell.Cell; + var CodeCell = codecell.CodeCell; + var Completer = completer.Completer; + var keycodes = keyboard.keycodes; + var specials = [ + keycodes.enter, + keycodes.esc, + keycodes.backspace, + keycodes.tab, + keycodes.up, + keycodes.down, + keycodes.left, + keycodes.right, + keycodes.shift, + keycodes.ctrl, + keycodes.alt, + keycodes.meta, + keycodes.capslock, + // keycodes.space, + keycodes.pageup, + keycodes.pagedown, + keycodes.end, + keycodes.home, + keycodes.insert, + keycodes.delete, + keycodes.numlock, + keycodes.f1, + keycodes.f2, + keycodes.f3, + keycodes.f4, + keycodes.f5, + keycodes.f6, + keycodes.f7, + keycodes.f8, + keycodes.f9, + keycodes.f10, + keycodes.f11, + keycodes.f12, + keycodes.f13, + keycodes.f14, + keycodes.f15 + ]; + + function loadCss(name) { + $('').attr({ + type: 'text/css', + rel: 'stylesheet', + href: requirejs.toUrl(name) + }).appendTo('head'); + } + + + function onlyModifierEvent(event) { + var key = keyboard.inv_keycodes[event.which]; + return ( + (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey) && + (key === 'alt' || key === 'ctrl' || key === 'meta' || key === 'shift') + ); + } + + function requestComplterServer(requestData, isAsync, handleResData) { + var serverUrl = config.remote_server_url ? config.remote_server_url : baseUrl; + if (serverUrl.charAt(serverUrl.length - 1) == '/') { + serverUrl += 'tabnine'; + } else { + serverUrl += '/tabnine'; + } + // use get to solve post redirecting too many times + $.get(serverUrl, { 'data': JSON.stringify(requestData) }) + .done(function (data) { + handleResData(data); + }).fail(function (error) { + console.log(logPrefix, ' get error: ', error); + }); + } + + function isValidCodeLine(line) { + // comment line is valid, since we want to get completions + if (line.length === 0 || + line.charAt(0) === '!') { + return false; + } + return true; + } + + // A Deep Completer which extends Completer + const DeepCompleter = function (cell, events) { + Completer.call(this, cell, events); + } + DeepCompleter.prototype = Object.create(Completer.prototype); + DeepCompleter.prototype.constructor = DeepCompleter; + DeepCompleter.prototype.finish_completing = function (msg) { + var optionsLimit = config.options_limit; + var beforeLineLimit = config.before_line_limit > 0 ? config.before_line_limit : Infinity; + var afterLineLimit = config.after_line_limit > 0 ? config.after_line_limit : Infinity; + if (this.visible && $('#complete').length) { + console.info(logPrefix, 'complete is visible, ignore by just return'); + return; + } + + var currEditor = this.editor; + var currCell = this.cell; + // check whether current cell satisfies line before and line after + var cursor = currEditor.getCursor(); + var currCellLines = currEditor.getValue().split("\n"); + var before = []; + var after = []; + var currLine = currCellLines[cursor.line]; + if (isValidCodeLine(currLine)) { + before.push(currLine.slice(0, cursor.ch)); + after.push(currLine.slice(cursor.ch, currLine.length)); + } + + var i = cursor.line - 1; + for (; i >= 0; before.length < beforeLineLimit, i--) { + if (isValidCodeLine(currCellLines[i])) { + before.push(currCellLines[i]); + } + } + requestInfo.request.Autocomplete.region_includes_beginning = (i < 0); + + i = cursor.line + 1; + for (; i < currCellLines.length && after.length < afterLineLimit; i++) { + if (isValidCodeLine(currCellLines[i])) { + after.push(currCellLines[i]); + } + } + + var cells = Jupyter.notebook.get_cells(); + var index; + for (index = cells.length - 1; index >= 0 && cells[index] != currCell; index--); + var regionIncludesBeginning = requestInfo.request.Autocomplete.region_includes_beginning; + requestInfo.request.Autocomplete.region_includes_beginning = regionIncludesBeginning && (index == 0); + requestInfo.request.Autocomplete.region_includes_end = (i == currCellLines.length) + && (index == cells.length - 1); + // need lookup other cells + if (before.length < beforeLineLimit || after.length < afterLineLimit) { + i = index - 1; + // always use for loop instead of while loop if poosible. + // since I always foget to describe/increase i in while loop + var atLineBeginning = true; // set true in case of three is no more lines before + for (; i >= 0 && before.length < beforeLineLimit; i--) { + var cellLines = cells[i].get_text().split("\n"); + var j = cellLines.length - 1; + atLineBeginning = false; + for (; j >= 0 && before.length < beforeLineLimit; j--) { + if (isValidCodeLine(cellLines[j])) { + before.push(cellLines[j]); + } + } + atLineBeginning = (j < 0); + } + // at the first cell and at the first line of that cell + requestInfo.request.Autocomplete.region_includes_beginning = (i < 0) && atLineBeginning; + + i = index + 1; + var atLineEnd = true; // set true in case of three is no more liens left + for (; i < cells.length && after.length < afterLineLimit; i++) { + var cellLines = cells[i].get_text().split("\n"); + j = 0; + atLineEnd = false; + for (; j < cellLines.length && after.length < afterLineLimit; j++) { + if (isValidCodeLine(cellLines[j])) { + after.push(cellLines[j]); + } + } + atLineEnd = (j == cellLines.length); + } + // at the last cell and at the last line of that cell + requestInfo.request.Autocomplete.region_includes_end = (i == cells.length) && atLineEnd; + } + before.reverse(); + this.before = before; + this.after = after; + + requestInfo.request.Autocomplete.before = before.join("\n"); + requestInfo.request.Autocomplete.after = after.join("\n"); + + this.complete = $('
').addClass('completions complete-dropdown-content'); + this.complete.attr('id', 'complete'); + $('body').append(this.complete); + this.visible = true; + // fix page flickering + this.start = currEditor.indexFromPos(cursor); + this.complete.css({ + 'display': 'none', + }); + + var that = this; + requestComplterServer(requestInfo, true, function (data) { + var complete = that.complete; + if (data.results.length == 0) { + that.close(); + return; + } + that.completions = data.results.slice(0, optionsLimit); + that.completions.forEach(function (res) { + var completeContainer = generateCompleteContainer(res); + complete.append(completeContainer); + }); + that.add_user_msg(data.user_message); + that.set_location(data.old_prefix); + that.add_keyevent_listeners() + }); + return true; + } + + DeepCompleter.prototype.add_user_msg = function (user_messages) { + var that = this; + if (user_messages) { + user_messages.forEach(function (user_message) { + var msgLine = $('
').addClass('user-message'); + $('').text(user_message).appendTo(msgLine); + that.complete.append(msgLine); + }); + } + } + + DeepCompleter.prototype.update = function () { + // In this case, only current line have been changed. + // so we can use cached other lines and this line to + // generate before and after + var optionsLimit = config.options_limit; + if (!this.complete) { + return; + } + var cursor = this.editor.getCursor(); + this.start = this.editor.indexFromPos(cursor); // get current cursor + var currLineText = this.editor.getLineHandle(cursor.line).text; + var currLineBefore = currLineText.slice(0, cursor.ch); + var currLineAfter = currLineText.slice(cursor.ch, currLineText.length); + if (this.before.length > 0) { + this.before[this.before.length - 1] = currLineBefore; + } else { + this.before.push(currLineBefore); + } + if (this.after.length > 0) { + this.after[0] = currLineAfter; + } else { + this.after.push(currLineAfter); + } + requestInfo.request.Autocomplete.before = this.before.join('\n'); + requestInfo.request.Autocomplete.after = this.after.join('\n'); + var that = this; + requestComplterServer(requestInfo, true, function (data) { + if (data.results.length == 0) { + that.close(); + return; + } + var results = data.results; + var completeContainers = $("#complete").find('.complete-container'); + var i; + that.completions = results.slice(0, optionsLimit); + // replace current options first + for (i = 0; i < that.completions.length && i < completeContainers.length; i++) { + $(completeContainers[i]).find('.complete-word').text(results[i].new_prefix); + $(completeContainers[i]).find('.complete-detail').text(results[i].detail); + } + // add + for (; i < that.completions.length; i++) { + var completeContainer = generateCompleteContainer(results[i]); + that.complete.append(completeContainer); + } + // remove + for (; i < completeContainers.length; i++) { + $(completeContainers[i]).remove(); + } + + var userMessages = $('#complete').find('.user-message'); + if (userMessages) { + if (userMessages instanceof Array) { + for (var i = 0; i < userMessages.length; i++) { + $(userMessages[i]).remove(); + } + } else { + $(userMessages).remove(); + } + } + that.add_user_msg(data.user_message); + + that.set_location(data.old_prefix); + that.editor.off('keydown', that._handle_keydown); + that.editor.off('keyup', that._handle_keyup); + that.add_keyevent_listeners(); + }); + }; + + DeepCompleter.prototype.close = function () { + this.done = true; + $('#complete').remove(); + this.editor.off('keydown', this._handle_keydown); + this.visible = false; + this.completions = null; + this.completeFrom = null; + this.complete = null; + // before are copied from completer.js + this.editor.off('keyup', this._handle_key_up); + }; + + DeepCompleter.prototype.set_location = function (oldPrefix) { + if (!this.complete) { + return; + } + var start = this.start; + this.completeFrom = this.editor.posFromIndex(start); + if (oldPrefix) { + oldPrefix = oldPrefix; + this.completeFrom.ch -= oldPrefix.length; + // this.completeFrom.ch = Math.max(this.completeFrom.ch, 0); + } + var pos = this.editor.cursorCoords( + this.completeFrom + ); + + var left = pos.left - 3; + var top; + var cheight = this.complete.height(); + var wheight = $(window).height(); + if (pos.bottom + cheight + 5 > wheight) { + top = pos.top - cheight - 4; + } else { + top = pos.bottom + 1; + } + this.complete.css({ + 'left': left + 'px', + 'top': top + 'px', + 'display': 'initial' + }); + }; + + DeepCompleter.prototype.add_keyevent_listeners = function () { + var options = $("#complete").find('.complete-container'); + var editor = this.editor; + var currIndex = -1; + var preIndex; + this.isKeyupFired = true; // make keyup only fire once + var that = this; + this._handle_keydown = function (comp, event) { // define as member method to handle close + // since some opration is async, it's better to check whether complete is existing or not. + if (!$('#complete').length || !that.completions) { + // editor.off('keydown', this._handle_keydown); + // editor.off('keyup', this._handle_handle_keyup); + return; + } + that.isKeyupFired = false; + if (event.keyCode == keycodes.up || event.keyCode == keycodes.tab + || event.keyCode == keycodes.down || event.keyCode == keycodes.enter) { + event.codemirrorIgnore = true; + event._ipkmIgnore = true; + event.preventDefault(); + // it's better to prevent enter key when completions being shown + if (event.keyCode == keycodes.enter) { + that.close(); + return; + } + preIndex = currIndex; + currIndex = event.keyCode == keycodes.up ? currIndex - 1 : currIndex + 1; + currIndex = currIndex < 0 ? + options.length - 1 + : (currIndex >= options.length ? + currIndex - options.length + : currIndex); + $(options[currIndex]).css('background', 'lightblue'); + var end = editor.getCursor(); + if (that.completions[currIndex].old_suffix) { + end.ch += that.completions[currIndex].old_suffix.length; + } + var replacement = that.completions[currIndex].new_prefix; + replacement += that.completions[currIndex].new_suffix; + editor.replaceRange(replacement, that.completeFrom, end); + if (preIndex != -1) { + $(options[preIndex]).css('background', ''); + } + } else if (needUpdateComplete(event.keyCode)) { + // Let this be handled by keyup, since it can get current pressed key. + } else { + that.close(); + } + } + + var that = this; + this._handle_keyup = function (cmp, event) { + if (!that.isKeyupFired && !event.altKey && + !event.ctrlKey && !event.metaKey && needUpdateComplete(event.keyCode)) { + that.update(); + that.isKeyupFired = true; + }; + }; + + editor.on('keydown', this._handle_keydown); + editor.on('keyup', this._handle_keyup); + }; + + function generateCompleteContainer(responseComplete) { + var completeContainer = $('
') + .addClass('complete-container'); + var wordContainer = $('
') + .addClass('complete-block') + .addClass('complete-word') + .text(responseComplete.new_prefix); + completeContainer.append(wordContainer); + var probContainer = $('
') + .addClass('complete-block') + .addClass('complete-detail') + .text(responseComplete.detail) + completeContainer.append(probContainer); + return completeContainer; + } + + function isAlphabeticKeyCode(keyCode) { + return keyCode >= 65 && keyCode <= 90; + } + + function isNumberKeyCode(keyCode) { + return (keyCode >= 48 && keyCode <= 57) || (keyCode >= 96 && keyCode <= 105); + } + + function isOperatorKeyCode(keyCode) { + return (keyCode >= 106 && keyCode <= 111) || + (keyCode >= 186 && keyCode <= 192) || + (keyCode >= 219 && keyCode <= 222); + } + + function needUpdateComplete(keyCode) { + return isAlphabeticKeyCode(keyCode) || isNumberKeyCode(keyCode) || isOperatorKeyCode(keyCode); + } + + function patchCellKeyevent() { + var origHandleCodemirrorKeyEvent = Cell.prototype.handle_codemirror_keyevent; + Cell.prototype.handle_codemirror_keyevent = function (editor, event) { + if (!this.base_completer) { + console.log(logPrefix, ' new base completer'); + this.base_completer = new Completer(this, this.events); + } + + if (!this.deep_completer) { + console.log(logPrefix, ' new deep completer'); + this.deep_completer = new DeepCompleter(this, this.events) + } + + if (assistActive && !event.altKey && !event.metaKey && !event.ctrlKey + && (this instanceof CodeCell) && !onlyModifierEvent(event)) { + this.tooltip.remove_and_cancel_tooltip(); + if (!editor.somethingSelected() && + editor.getSelections().length <= 1 && + !this.completer.visible && + specials.indexOf(event.keyCode) == -1) { + var cell = this; + if (event.keyCode == keycodes.space && event.shiftKey) { + event.preventDefault(); + console.log(logPrefix, ' call base completer....'); + cell.completer = cell.base_completer; + } else { + console.log(logPrefix, ' call deep completer....'); + cell.completer = cell.deep_completer; + } + setTimeout(function () { + cell.completer.startCompletion(); + }, config.assist_delay); + } + } + return origHandleCodemirrorKeyEvent.apply(this, arguments); + }; + } + + function setAssistState(newState) { + assistActive = newState; + $('.assistant-toggle > .fa').toggleClass('fa-check', assistActive); + console.log(logPrefix, 'continuous autocompletion', assistActive ? 'on' : 'off'); + } + + function toggleAutocompletion() { + setAssistState(!assistActive); + } + + function addMenuItem() { + if ($('#help_menu').find('.assistant-toggle').length > 0) { + return; + } + var menuItem = $('
  • ').insertAfter('#keyboard_shortcuts'); + var menuLink = $('').text('Jupyter TabNine') + .addClass('assistant-toggle') + .attr('title', 'Provide continuous code autocompletion') + .on('click', toggleAutocompletion) + .appendTo(menuItem); + $('').addClass('fa menu-icon pull-right').prependTo(menuLink); + } + + + function load_notebook_extension() { + return Jupyter.notebook.config.loaded.then(function on_success() { + $.extend(true, config, Jupyter.notebook.config.data.jupyter_tabnine); + loadCss('./main.css'); + }, function on_error(err) { + console.warn(logPrefix, 'error loading config:', err); + }).then(function on_success() { + patchCellKeyevent(); + addMenuItem(); + setAssistState(config.assist_active); + }); + } + return { + load_ipython_extension: load_notebook_extension, + load_jupyter_extension: load_notebook_extension + }; +}); diff --git a/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/tabnine.yaml b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/tabnine.yaml new file mode 100644 index 000000000..9e31d4f63 --- /dev/null +++ b/src/jupyter_contrib_nbextensions/nbextensions/jupyter_tabnine/tabnine.yaml @@ -0,0 +1,47 @@ +Type: Jupyter Notebook Extension +Main: main.js +Name: Jupyter TabNine +Link: README.md +Description: | + Provide code auto-completion with deep learning for every keypress. +Compatibility: 4.x, 5.x +Parameters: +- name: jupyter_tabnine.before_line_limit + description: | + maximum number of lines before for context generation, + too many lines will slow down the request. -1 means Infinity, + thus the lines will equal to number of lines before current line. + input_type: number + default: 10 +- name: jupyter_tabnine.after_line_limit + description: | + maximum number of lines after for context generation, + too many lines will slow down the request. -1 means Infinity, + thus the lines will equal to number of lines after current line. + input_type: number + default: 10 +- name: jupyter_tabnine.options_limit + description: | + maximum number of options that will be shown + input_type: number + default: 10 +- name: jupyter_tabnine.assist_active + description: | + Enable continuous code auto-completion when notebook is first opened, or + if false, only when selected from extensions menu. + input_type: checkbox + default: true +- name: jupyter_tabnine.assist_delay + description: | + delay in milliseconds between keypress & completion request. + input_type: number + min: 0 + step: 1 + default: 0 +- name: jupyter_tabnine.remote_server_url + description: | + remote server url, you may want to use a remote server to handle client request. + this can spped up the request handling depending on the server configuration. + refer to https://github.com/wenmin-wu/jupyter-tabnine to see how to deploy remote server. + input_type: string + default: '' diff --git a/src/jupyter_tabnine/__init__.py b/src/jupyter_tabnine/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/jupyter_tabnine/handler.py b/src/jupyter_tabnine/handler.py new file mode 100644 index 000000000..a04ccdc1e --- /dev/null +++ b/src/jupyter_tabnine/handler.py @@ -0,0 +1,14 @@ +from tornado import web +from urllib.parse import unquote +from notebook.base.handlers import IPythonHandler + +class TabNineHandler(IPythonHandler): + def initialize(self, tabnine): + self.tabnine = tabnine + + @web.authenticated + async def get(self): + url_params = self.request.uri + request_data = unquote(url_params[url_params.index('=')+1:]) + response = self.tabnine.request(request_data) + self.write(response) diff --git a/src/jupyter_tabnine/tabnine.py b/src/jupyter_tabnine/tabnine.py new file mode 100644 index 000000000..62ef50358 --- /dev/null +++ b/src/jupyter_tabnine/tabnine.py @@ -0,0 +1,190 @@ +import json +import logging +import os +import platform +import subprocess +import threading +from urllib.request import urlopen +from urllib.error import HTTPError + +if platform.system() == "Windows": + try: + from colorama import init + init(convert=True) + except ImportError: + try: + import pip + pip.main(['install', '--user', 'colorama']) + from colorama import init + init(convert=True) + except Exception: + logger = logging.getLogger('ImportError') + logger.error('Install colorama failed. Install it manually to enjoy colourful log.') + + +logging.basicConfig(level=logging.INFO, + format='\x1b[1m\x1b[33m[%(levelname)s %(asctime)s.%(msecs)03d %(name)s]\x1b[0m: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + +_TABNINE_UPDATE_VERSION_URL = "https://update.tabnine.com/version" +_TABNINE_DOWNLOAD_URL_FORMAT = "https://update.tabnine.com/{}" +_SYSTEM_MAPPING = { + "Darwin": "apple-darwin", + "Linux": "unknown-linux-gnu", + "Windows": "pc-windows-gnu", +} + +class TabNineDownloader(threading.Thread): + def __init__(self, download_url, output_path): + threading.Thread.__init__(self) + self.download_url = download_url + self.output_path = output_path + self.logger = logging.getLogger(self.__class__.__name__) + + def run(self): + output_dir = os.path.dirname(self.output_path) + try: + self.logger.info('Begin to download TabNine Binary from %s', + self.download_url) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + with urlopen(self.download_url) as res, \ + open(self.output_path, 'wb') as out: + out.write(res.read()) + os.chmod(self.output_path, 0o755) + self.logger.info('Finish download TabNine Binary to %s', + self.output_path) + except Exception as e: + self.logger.error("Download failed, error: %s", e) + + +class TabNine(object): + """ + TabNine python wrapper + """ + def __init__(self): + self.name = "tabnine" + self._proc = None + self._response = None + self.logger = logging.getLogger(self.__class__.__name__) + self._install_dir = os.path.dirname(os.path.realpath(__file__)) + self._binary_dir = os.path.join(self._install_dir, "binaries") + self.logger.info(" install dir: %s", self._install_dir) + self.download_if_needed() + + def request(self, data): + proc = self._get_running_tabnine() + if proc is None: + return + try: + proc.stdin.write((data + "\n").encode("utf8")) + proc.stdin.flush() + except BrokenPipeError: + self._restart() + return + + output = proc.stdout.readline().decode("utf8") + try: + return json.loads(output) + except json.JSONDecodeError: + self.logger.debug("Tabnine output is corrupted: " + output) + + def _restart(self): + if self._proc is not None: + self._proc.terminate() + self._proc = None + path = get_tabnine_path(self._binary_dir) + if path is None: + self.logger.error("no TabNine binary found") + return + self._proc = subprocess.Popen( + [ + path, + "--client", + "sublime", + "--log-file-path", + os.path.join(self._install_dir, "tabnine.log"), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + def _get_running_tabnine(self): + if self._proc is None: + self._restart() + if self._proc is not None and self._proc.poll(): + self.logger.error( + "TabNine exited with code {}".format(self._proc.returncode) + ) + self._restart() + return self._proc + + def download_if_needed(self): + if os.path.isdir(self._binary_dir): + tabnine_path = get_tabnine_path(self._binary_dir) + if tabnine_path is not None: + os.chmod(tabnine_path, 0o755) + self.logger.info( + "TabNine binary already exists in %s ignore downloading", + tabnine_path + ) + return + self._download() + + def _download(self): + tabnine_sub_path = get_tabnine_sub_path() + binary_path = os.path.join(self._binary_dir, tabnine_sub_path) + download_url = _TABNINE_DOWNLOAD_URL_FORMAT.format(tabnine_sub_path) + TabNineDownloader(download_url, binary_path).start() + + +def get_tabnine_sub_path(): + version = get_tabnine_version() + architect = parse_architecture(platform.machine()) + system = _SYSTEM_MAPPING[platform.system()] + execute_name = executable_name("TabNine") + return "{}/{}-{}/{}".format(version, architect, system, execute_name) + + +def get_tabnine_version(): + try: + version = urlopen(_TABNINE_UPDATE_VERSION_URL).read().decode("UTF-8").strip() + return version + except HTTPError: + return None + + +def get_tabnine_path(binary_dir): + versions = os.listdir(binary_dir) + versions.sort(key=parse_semver, reverse=True) + for version in versions: + triple = "{}-{}".format( + parse_architecture(platform.machine()), _SYSTEM_MAPPING[platform.system()] + ) + path = os.path.join(binary_dir, version, triple, executable_name("TabNine")) + if os.path.isfile(path): + return path + return None + + +# Adapted from the sublime plugin +def parse_semver(s): + try: + return [int(x) for x in s.split(".")] + except ValueError: + return [] + + +def parse_architecture(arch): + if arch == "AMD64": + return "x86_64" + else: + return arch + + +def executable_name(name): + if platform.system() == "Windows": + return name + ".exe" + else: + return name