Upgrade xeus5 (#624)

* Upgraded to xeus 5 and removed xtl * Disabled C++ tests until complete rewriting --------- Co-authored-by: Isabel Paredes <[email protected]>
jupyter-xeus · May 29, 2024 · 47cf95c · 47cf95c
1 parent f9f8a05
commit 47cf95c
Show file tree

Hide file tree

Showing 22 changed files with 169 additions and 119 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -106,16 +106,12 @@ endif()
 # Dependencies
 # ============
 
-set(xtl_REQUIRED_VERSION 0.7.0)
 set(xeus-zmq_REQUIRED_VERSION 1.2.0)
 set(xeus-lite_REQUIRED_VERSION 1.0.1)
 set(pybind11_REQUIRED_VERSION 2.6.1)
 set(pybind11_json_REQUIRED_VERSION 0.2.8)
 set(pyjs_REQUIRED_VERSION 2.0.0)
 
-if (NOT TARGET xtl)
-    find_package(xtl ${xtl_REQUIRED_VERSION} REQUIRED)
-endif ()
 if (NOT TARGET pybind11::headers)
     find_package(pybind11 ${pybind11_REQUIRED_VERSION} REQUIRED)
 endif ()
@@ -304,7 +300,7 @@ macro(xpyt_create_target target_name src headers linkage output_name)
         set(XPYT_XEUS_TARGET xeus-zmq-static)
     endif ()
 
-    target_link_libraries(${target_name} PUBLIC ${XPYT_XEUS_TARGET} xtl PRIVATE pybind11::pybind11 pybind11_json)
+    target_link_libraries(${target_name} PUBLIC ${XPYT_XEUS_TARGET} PRIVATE pybind11::pybind11 pybind11_json)
     if (WIN32 OR CYGWIN)
         target_link_libraries(${target_name} PRIVATE ${PYTHON_LIBRARIES})
     elseif (APPLE)

diff --git a/environment-dev.yml b/environment-dev.yml
@@ -7,10 +7,9 @@ dependencies:
   - cxx-compiler
   - ninja
   # Host dependencies
-  - xeus-zmq>=1.3.0,<2.0
-  - nlohmann_json
-  - cppzmq
-  - xtl>=0.7, <0.8
+  - xeus>=5.0.0
+  - xeus-zmq>=3.0,<4.0
+  - nlohmann_json=3.11.3
   - pybind11>=2.6.1,<3.0
   - pybind11_json>=0.2.6,<0.3
   - xeus-python-shell>=0.5.0,<0.6

diff --git a/environment-wasm-host.yml b/environment-wasm-host.yml
@@ -6,7 +6,6 @@ dependencies:
   - ipython
   - pybind11
   - jedi
-  - xtl
   - nlohmann_json
   - pybind11_json
   - numpy
@@ -15,4 +14,4 @@ dependencies:
   - xeus-python-shell>=0.5
   - pyjs>=2.0.0
   - requests-wasm-polyfill>=0.3.0
-  - libpython
+  - libpython
diff --git a/include/xeus-python/xdebugger.hpp b/include/xeus-python/xdebugger.hpp
@@ -20,9 +20,7 @@
 #include <mutex>
 #include <set>
 
-#include "zmq.hpp"
 #include "nlohmann/json.hpp"
-#include "xeus/xeus_context.hpp"
 #include "pybind11/pybind11.h"
 #include "xeus-zmq/xdebugger_base.hpp"
 #include "xeus_python_config.hpp"
@@ -40,7 +38,7 @@ namespace xpyt
 
         using base_type = xeus::xdebugger_base;
 
-        debugger(zmq::context_t& context,
+        debugger(xeus::xcontext& context,
                  const xeus::xconfiguration& config,
                  const std::string& user_name,
                  const std::string& session_id,
@@ -59,10 +57,8 @@ namespace xpyt
         nl::json variables_request_impl(const nl::json& message) override;
 
         bool start_debugpy();
-        bool start(zmq::socket_t& header_socket,
-                   zmq::socket_t& request_socket) override;
-        void stop(zmq::socket_t& header_socket,
-                  zmq::socket_t& request_socket) override;
+        bool start() override;
+        void stop() override;
         xeus::xdebugger_info get_debugger_info() const override;
         std::string get_cell_temporary_file(const std::string& code) const override;
 

diff --git a/include/xeus-python/xinterpreter.hpp b/include/xeus-python/xinterpreter.hpp
@@ -51,12 +51,11 @@ namespace xpyt
 
         void configure_impl() override;
 
-        nl::json execute_request_impl(int execution_counter,
-                                      const std::string& code,
-                                      bool silent,
-                                      bool store_history,
-                                      nl::json user_expressions,
-                                      bool allow_stdin) override;
+        void execute_request_impl(send_reply_callback cb,
+                                  int execution_counter,
+                                  const std::string& code,
+                                  xeus::execute_request_config config,
+                                  nl::json user_expressions) override;
 
         nl::json complete_request_impl(const std::string& code, int cursor_pos) override;
 
@@ -72,6 +71,9 @@ namespace xpyt
 
         nl::json internal_request_impl(const nl::json& content) override;
 
+        void set_request_context(xeus::xrequest_context context) override;
+        const xeus::xrequest_context& get_request_context() const noexcept override;
+
         void redirect_output();
 
         py::object m_ipython_shell_app;

diff --git a/include/xeus-python/xinterpreter_raw.hpp b/include/xeus-python/xinterpreter_raw.hpp
@@ -49,12 +49,11 @@ namespace xpyt
 
         void configure_impl() override;
 
-        nl::json execute_request_impl(int execution_counter,
-                                      const std::string& code,
-                                      bool silent,
-                                      bool store_history,
-                                      nl::json user_expressions,
-                                      bool allow_stdin) override;
+        void execute_request_impl(send_reply_callback cb,
+                                  int execution_counter,
+                                  const std::string& code,
+                                  xeus::execute_request_config config,
+                                  nl::json user_expressions) override;
 
         nl::json complete_request_impl(const std::string& code, int cursor_pos) override;
 
@@ -68,6 +67,9 @@ namespace xpyt
 
         void shutdown_request_impl() override;
 
+        void set_request_context(xeus::xrequest_context context) override;
+        const xeus::xrequest_context& get_request_context() const noexcept override;
+
         void redirect_output();
 
         py::object m_displayhook;

diff --git a/src/main.cpp b/src/main.cpp
@@ -21,12 +21,12 @@
 #include <unistd.h>
 #endif
 
-#include "xeus/xeus_context.hpp"
 #include "xeus/xkernel.hpp"
 #include "xeus/xkernel_configuration.hpp"
 #include "xeus/xinterpreter.hpp"
 
-#include "xeus-zmq/xserver_shell_main.hpp"
+#include "xeus-zmq/xserver_zmq_split.hpp"
+#include "xeus-zmq/xzmq_context.hpp"
 
 #include "pybind11/embed.h"
 #include "pybind11/pybind11.h"
@@ -99,9 +99,7 @@ int main(int argc, char* argv[])
     }
     delete[] argw;
 
-    using context_type = xeus::xcontext_impl<zmq::context_t>;
-    using context_ptr = std::unique_ptr<context_type>;
-    context_ptr context = context_ptr(new context_type());
+    std::unique_ptr<xeus::xcontext> context = xeus::make_zmq_context();
 
     // Instantiating the xeus xinterpreter
     bool raw_mode = xpyt::extract_option("-r", "--raw", argc, argv);
@@ -156,6 +154,7 @@ int main(int argc, char* argv[])
     }
     else
     {
+        std::clog << "Instantiating kernel" << std::endl;
         xeus::xkernel kernel(xeus::get_user_name(),
                              std::move(context),
                              std::move(interpreter),
@@ -165,6 +164,7 @@ int main(int argc, char* argv[])
                              xpyt::make_python_debugger,
                              debugger_config);
 
+        std::cout << "Getting config" << std::endl;
         const auto& config = kernel.get_config();
         std::clog <<
             "Starting xeus-python kernel...\n\n"

diff --git a/src/xdebugger.cpp b/src/xdebugger.cpp
@@ -42,7 +42,7 @@ using namespace std::placeholders;
 
 namespace xpyt
 {
-    debugger::debugger(zmq::context_t& context,
+    debugger::debugger(xeus::xcontext& context,
                        const xeus::xconfiguration& config,
                        const std::string& user_name,
                        const std::string& session_id,
@@ -305,7 +305,7 @@ namespace xpyt
         return status == "ok";
     }
 
-    bool debugger::start(zmq::socket_t& header_socket, zmq::socket_t& request_socket)
+    bool debugger::start()
     {
         std::string temp_dir = xeus::get_temp_directory_path();
         std::string log_dir = temp_dir + "/" + "xpython_debug_logs_" + std::to_string(xeus::get_current_pid());
@@ -322,8 +322,7 @@ namespace xpyt
         std::string controller_header_end_point = xeus::get_controller_end_point("debugger_header");
         std::string publisher_end_point = xeus::get_publisher_end_point();
 
-        request_socket.bind(controller_end_point);
-        header_socket.bind(controller_header_end_point);
+        bind_sockets(controller_end_point, controller_header_end_point);
 
         std::string debugpy_end_point = "tcp://" + m_debugpy_host + ':' + m_debugpy_port;
         std::thread client(&xdap_tcp_client::start_debugger,
@@ -334,22 +333,19 @@ namespace xpyt
                            controller_header_end_point);
         client.detach();
 
-        request_socket.send(zmq::message_t("REQ", 3), zmq::send_flags::none);
-        zmq::message_t ack;
-        (void)request_socket.recv(ack);
+        send_recv_request("REQ");
 
-        std::string tmp_folder =  get_tmp_prefix();
+        std::string tmp_folder = get_tmp_prefix();
         xeus::create_directory(tmp_folder);
 
         return true;
     }
 
-    void debugger::stop(zmq::socket_t& header_socket, zmq::socket_t& request_socket)
+    void debugger::stop()
     {
         std::string controller_end_point = xeus::get_controller_end_point("debugger");
         std::string controller_header_end_point = xeus::get_controller_end_point("debugger_header");
-        request_socket.unbind(controller_end_point);
-        header_socket.unbind(controller_header_end_point);
+        unbind_sockets(controller_end_point, controller_header_end_point);
     }
 
     xeus::xdebugger_info debugger::get_debugger_info() const
@@ -373,7 +369,7 @@ namespace xpyt
                                                           const std::string& session_id,
                                                           const nl::json& debugger_config)
     {
-        return std::unique_ptr<xeus::xdebugger>(new debugger(context.get_wrapped_context<zmq::context_t>(),
+        return std::unique_ptr<xeus::xdebugger>(new debugger(context,
                                                              config, user_name, session_id, debugger_config));
     }
 }
diff --git a/src/xdebugpy_client.cpp b/src/xdebugpy_client.cpp
@@ -8,7 +8,6 @@
 * The full license is in the file LICENSE, distributed with this software. *
 ****************************************************************************/
 
-#include "zmq_addon.hpp"
 #include "nlohmann/json.hpp"
 #include "xeus/xmessage.hpp"
 #include "xdebugpy_client.hpp"
@@ -20,7 +19,7 @@ namespace nl = nlohmann;
 
 namespace xpyt
 {
-    xdebugpy_client::xdebugpy_client(zmq::context_t& context,
+    xdebugpy_client::xdebugpy_client(xeus::xcontext& context,
                                      const xeus::xconfiguration& config,
                                      int socket_linger,
                                      const xdap_tcp_configuration& dap_config,

diff --git a/src/xdebugpy_client.hpp b/src/xdebugpy_client.hpp
@@ -25,7 +25,7 @@ namespace xpyt
         using base_type = xdap_tcp_client;
         using event_callback = base_type::event_callback;
 
-        xdebugpy_client(zmq::context_t& context,
+        xdebugpy_client(xeus::xcontext& context,
                         const xeus::xconfiguration& config,
                         int socket_linger,
                         const xdap_tcp_configuration& dap_config,

diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp
@@ -109,14 +109,15 @@ namespace xpyt
         {
             redirect_output();
         }
+
+        py::module context_module = get_request_context_module();
     }
 
-    nl::json interpreter::execute_request_impl(int /*execution_count*/,
-                                               const std::string& code,
-                                               bool silent,
-                                               bool store_history,
-                                               nl::json user_expressions,
-                                               bool allow_stdin)
+    void interpreter::execute_request_impl(send_reply_callback cb,
+                                           int /*execution_count*/,
+                                           const std::string& code,
+                                           xeus::execute_request_config config,
+                                           nl::json user_expressions)
     {
         py::gil_scoped_acquire acquire;
         nl::json kernel_res;
@@ -126,23 +127,28 @@ namespace xpyt
 
         // Scope guard performing the temporary monkey patching of input and
         // getpass with a function sending input_request messages.
-        auto input_guard = input_redirection(allow_stdin);
+        auto input_guard = input_redirection(config.allow_stdin);
 
         bool exception_occurred = false;
-        try{
-            m_ipython_shell.attr("run_cell")(code, "store_history"_a=store_history, "silent"_a=silent);
+        try
+        {
+            m_ipython_shell.attr("run_cell")(code, "store_history"_a=config.store_history, "silent"_a=config.silent);
         }
-        catch(std::runtime_error& e){
+        catch(std::runtime_error& e)
+        {
             const std::string error_msg = e.what();
-            if(!silent){
+            if(!config.silent)
+            {
                 publish_execution_error("RuntimeError", error_msg, std::vector<std::string>());
             }
             kernel_res["ename"] = "std::runtime_error";
             kernel_res["evalue"] = error_msg;
             exception_occurred = true;
         }
-        catch(...){
-            if(!silent){
+        catch(...)
+        {
+            if(!config.silent)
+            {
                 publish_execution_error("unknown_error", "", std::vector<std::string>());
             }
             kernel_res["ename"] = "UnknownError";
@@ -154,10 +160,12 @@ namespace xpyt
         kernel_res["payload"] = m_ipython_shell.attr("payload_manager").attr("read_payload")();
         m_ipython_shell.attr("payload_manager").attr("clear_payload")();
 
-        if(exception_occurred){
+        if(exception_occurred)
+        {
             kernel_res["status"] = "error";
             kernel_res["traceback"] = std::vector<std::string>();
-            return kernel_res;
+            cb(kernel_res);
+            return;
         }
 
         if (m_ipython_shell.attr("last_error").is_none())
@@ -171,7 +179,7 @@ namespace xpyt
 
             xerror error = extract_error(pyerror);
 
-            if (!silent)
+            if (!config.silent)
             {
                 publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback);
             }
@@ -181,8 +189,7 @@ namespace xpyt
             kernel_res["evalue"] = error.m_evalue;
             kernel_res["traceback"] = error.m_traceback;
         }
-
-        return kernel_res;
+        cb(kernel_res);
     }
 
     nl::json interpreter::complete_request_impl(
@@ -347,6 +354,21 @@ namespace xpyt
         return reply;
     }
 
+    void interpreter::set_request_context(xeus::xrequest_context context)
+    {
+        py::gil_scoped_acquire acquire;
+        py::module context_module = get_request_context_module();
+        context_module.attr("set_request_context")(context);
+    }
+
+    const xeus::xrequest_context& interpreter::get_request_context() const noexcept
+    {
+        py::gil_scoped_acquire acquire;
+        py::module context_module = get_request_context_module();
+        py::object res = context_module.attr("get_request_context")();
+        return *(res.cast<xeus::xrequest_context*>());
+    }
+
     void interpreter::redirect_output()
     {
         py::module sys = py::module::import("sys");