Skip to content

Convert Python bytes and strings to Elixir binaries without copying #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 7, 2025
Merged

Convert Python bytes and strings to Elixir binaries without copying #19

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f9b3e0
Convert Python bytes and strings to Elixir binaries without copying
jonatanklosko Mar 7, 2025
3d488be
Add comments
jonatanklosko Mar 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions c_src/pythonx/pythonx.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,16 +266,16 @@ void raise_if_failed(ErlNifEnv *env, Py_ssize_t size) {
}
}

ERL_NIF_TERM py_object_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) {
ERL_NIF_TERM py_str_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) {
Py_ssize_t size;
auto buffer = PyUnicode_AsUTF8AndSize(py_object, &size);
raise_if_failed(env, buffer);

ERL_NIF_TERM binary_term;
auto data = enif_make_new_binary(env, size, &binary_term);
memcpy(data, buffer, size);

return binary_term;
// The buffer is immutable and lives as long as the Python object,
// so we create the term as a resource binary to make it zero-copy.
Py_IncRef(py_object);
auto ex_object_resource = fine::make_resource<ExObjectResource>(py_object);
return fine::make_resource_binary(env, ex_object_resource, buffer, size);
}

fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path,
Expand Down Expand Up @@ -583,7 +583,7 @@ fine::Term unicode_to_string(ErlNifEnv *env, ExObject ex_object) {
ensure_initialized();
auto gil_guard = PyGILGuard();

return py_object_to_binary_term(env, ex_object.resource->py_object);
return py_str_to_binary_term(env, ex_object.resource->py_object);
}

FINE_NIF(unicode_to_string, ERL_NIF_DIRTY_JOB_CPU_BOUND);
Expand Down Expand Up @@ -745,7 +745,7 @@ fine::Term format_exception(ErlNifEnv *env, ExError error) {
auto py_line = PyList_GetItem(py_lines, i);
raise_if_failed(env, py_line);

terms.push_back(py_object_to_binary_term(env, py_line));
terms.push_back(py_str_to_binary_term(env, py_line));
}

return enif_make_list_from_array(env, terms.data(),
Expand Down Expand Up @@ -803,7 +803,7 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) {
raise_if_failed(env, py_str);
auto py_str_guard = PyDecRefGuard(py_str);

auto binary_term = py_object_to_binary_term(env, py_str);
auto binary_term = py_str_to_binary_term(env, py_str);

return fine::encode(
env, std::make_tuple(atoms::integer, fine::Term(binary_term)));
Expand Down Expand Up @@ -904,7 +904,7 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) {
auto is_unicode = PyObject_IsInstance(py_object, py_str_type);
raise_if_failed(env, is_unicode);
if (is_unicode) {
return py_object_to_binary_term(env, py_object);
return py_str_to_binary_term(env, py_object);
}

auto py_bytes_type = PyDict_GetItemString(py_builtins, "bytes");
Expand All @@ -917,11 +917,11 @@ fine::Term decode_once(ErlNifEnv *env, ExObject ex_object) {
auto result = PyBytes_AsStringAndSize(py_object, &buffer, &size);
raise_if_failed(env, result);

ERL_NIF_TERM binary_term;
auto data = enif_make_new_binary(env, size, &binary_term);
memcpy(data, buffer, size);

return binary_term;
// The buffer is immutable and lives as long as the Python object,
// so we create the term as a resource binary to make it zero-copy.
Py_IncRef(py_object);
auto ex_object_resource = fine::make_resource<ExObjectResource>(py_object);
return fine::make_resource_binary(env, ex_object_resource, buffer, size);
}

auto py_set_type = PyDict_GetItemString(py_builtins, "set");
Expand Down Expand Up @@ -1341,7 +1341,7 @@ eval(ErlNifEnv *env, ErlNifBinary code, std::string code_md5,
continue;
}

auto key_term = py_object_to_binary_term(env, py_key);
auto key_term = py_str_to_binary_term(env, py_key);
key_terms.push_back(key_term);

// Incref before making the resource
Expand Down