Merge pull request #9 from eriknw/cython_utils

Add Cython utils needed for zero-copy import and export.

Author: jim22k

.gitattributes

@@ -0,0 +1 @@
+suitesparse_graphblas/_version.py export-subst

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2021 Anaconda Inc., Graphegon, and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -0,0 +1,8 @@
+include setup.py
+include README.md
+include LICENSE
+include suitesparse_graphblas/*.pxd
+include suitesparse_graphblas/*.pyx
+include suitesparse_graphblas/*.h
+include versioneer.py
+include suitesparse_graphblas/_version.py

pyproject.toml

@@ -0,0 +1,5 @@
+[build-system]
+requires = ["setuptools", "wheel", "numpy>=1.15"]
+
+[tool.black]
+line-length = 100

setup.cfg

@@ -0,0 +1,37 @@
+[aliases]
+test=pytest
+
+[flake8]
+max-line-length = 100
+exclude =
+    versioneer.py,
+ignore =
+    E203,   # whitespace before ':'
+    E231,   # Multiple spaces around ","
+    W503,   # line break before binary operator
+
+[coverage:run]
+source = suitesparse_graphblas
+plugins = Cython.Coverage
+omit =
+    suitesparse_graphblas/_version.py
+
+[coverage:report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    pragma: no cover
+
+    raise AssertionError
+    raise NotImplementedError
+
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = suitesparse_graphblas/_version.py
+versionfile_build = suitesparse_graphblas/_version.py
+tag_prefix=
+parentdir_prefix=suitesparse_graphblas-
+
+[tool:pytest]
+testpaths = suitesparse_graphblas/tests
+

setup.py

@@ -1,14 +1,69 @@
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Extension
+from glob import glob
+
+try:
+    from Cython.Build import cythonize
+    from Cython.Compiler.Options import get_directive_defaults
+
+    use_cython = True
+except ImportError:
+    use_cython = False
+import numpy as np
+import os
+import sys
+import versioneer
+
+define_macros = [("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")]
+
+if use_cython:
+    suffix = ".pyx"
+    directive_defaults = get_directive_defaults()
+    directive_defaults["binding"] = True
+    directive_defaults["language_level"] = 3
+    if os.environ.get("CYTHON_COVERAGE"):
+        directive_defaults["linetrace"] = True
+        define_macros.append(("CYTHON_TRACE_NOGIL", "1"))
+else:
+    suffix = ".c"
+
+include_dirs = [np.get_include(), os.path.join(sys.prefix, "include")]
+ext_modules = [
+    Extension(
+        name[: -len(suffix)].replace("/", ".").replace("\\", "."),
+        [name],
+        include_dirs=include_dirs,
+        define_macros=define_macros,
+    )
+    for name in glob(f"suitesparse_graphblas/**/*{suffix}", recursive=True)
+]
+if use_cython:
+    ext_modules = cythonize(ext_modules, include_path=include_dirs)
+
+with open("README.md") as f:
+    long_description = f.read()
+
+package_data = {"suitesparse_graphblas": ["*.pyx", "*.pxd", "*.h"]}
+if sys.platform == "win32":
+    package_data["suitesparse_graphblas"].append("*.dll")
 
 setup(
-    name='suitesparse-graphblas',
-    version='4.0.3',
-    description='SuiteSparse:GraphBLAS Python bindings.',
+    name="suitesparse-graphblas",
+    version=versioneer.get_version(),
+    cmdclass=versioneer.get_cmdclass(),
+    description="SuiteSparse:GraphBLAS Python bindings.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
     packages=find_packages(),
-    author='Michel Pelletier, James Kitchen, Erik Welch',
+    author="Michel Pelletier, James Kitchen, Erik Welch",
+    author_email="michel@graphegon.com,jim22k@gmail.com,erik.n.welch@gmail.com",
+    url="https://github.com/GraphBLAS/python-suitesparse-graphblas",
+    ext_modules=ext_modules,
     cffi_modules=["suitesparse_graphblas/build.py:ffibuilder"],
-    install_requires=["cffi>=1.0.0"],
+    python_requires=">=3.7",
+    install_requires=["cffi>=1.0.0", "numpy>=1.15"],
     setup_requires=["cffi>=1.0.0", "pytest-runner"],
     tests_require=["pytest"],
+    license="Apache License 2.0",
+    package_data=package_data,
+    include_package_data=True,
 )
-

suitesparse_graphblas/__init__.py

@@ -1 +1,44 @@
-from ._graphblas import ffi, lib
+from ._graphblas import ffi, lib  # noqa
+from . import utils
+from ._version import get_versions
+
+
+def is_initialized():
+    """Is GraphBLAS initialized via GrB_init or GxB_init?"""
+    return lib.GxB_Global_Option_get(lib.GxB_MODE, ffi.new("GrB_Mode*")) != lib.GrB_PANIC
+
+
+def initialize(*, blocking=False, memory_manager="numpy"):
+    """Initialize GraphBLAS via GrB_init or GxB_init.
+
+    This must be called before any other GraphBLAS functions are called.
+    A RuntimeError will be raised if called more than once.
+
+    Parameters
+    ----------
+    blocking : bool, optional
+        Whether to call init with GrB_BLOCKING or GrB_NONBLOCKING.
+        Default is False.
+    memory_manager : {'numpy', 'c'}, optional
+        Choose which malloc/free functions to use.  'numpy' uses numpy's
+        allocators, which makes it safe to perform zero-copy to and from numpy,
+        and allows Python to track memory usage via tracemalloc (if enabled).
+        'c' uses the default allocators.  Default is 'numpy'.
+
+    The global variable `suitesparse_graphblas.is_initialized` indicates whether
+    GraphBLAS has been initialized.
+    """
+    if is_initialized():
+        raise RuntimeError("GraphBLAS is already initialized!  Unable to initialize again.")
+    blocking = lib.GrB_BLOCKING if blocking else lib.GrB_NONBLOCKING
+    memory_manager = memory_manager.lower()
+    if memory_manager == "numpy":
+        utils.call_gxb_init(ffi, lib, blocking)
+    elif memory_manager == "c":
+        lib.GrB_init(blocking)
+    else:
+        raise ValueError(f'memory_manager argument must be "numpy" or "c"; got: {memory_manager!r}')
+
+
+__version__ = get_versions()["version"]
+del get_versions

suitesparse_graphblas/_version.py

@@ -380,9 +380,7 @@ def get_groups(ast):
     seen.update(vals)
     groups["GrB typedef"] = sorted(vals, key=sort_key)
 
-    missing_typedefs = {
-        x for x in lines if "typedef" in x and "GB" in x and "(" not in x
-    } - seen
+    missing_typedefs = {x for x in lines if "typedef" in x and "GB" in x and "(" not in x} - seen
     assert not missing_typedefs
     assert all(x.endswith(";") for x in seen)  # sanity check
 
@@ -393,20 +391,14 @@ def get_groups(ast):
     vals = {x for x in enums if "} GrB" in x}
     for val in vals:
         seen.update(val.splitlines())
-    groups["GrB typedef enums"] = sorted(
-        vals, key=lambda x: sort_key(x.rsplit("}", 1)[-1])
-    )
+    groups["GrB typedef enums"] = sorted(vals, key=lambda x: sort_key(x.rsplit("}", 1)[-1]))
 
     vals = {x for x in enums if "} GxB" in x}
     for val in vals:
         seen.update(val.splitlines())
-    groups["GxB typedef enums"] = sorted(
-        vals, key=lambda x: sort_key(x.rsplit("}", 1)[-1])
-    )
+    groups["GxB typedef enums"] = sorted(vals, key=lambda x: sort_key(x.rsplit("}", 1)[-1]))
 
-    missing_enums = (
-        set(enums) - set(groups["GrB typedef enums"]) - set(groups["GxB typedef enums"])
-    )
+    missing_enums = set(enums) - set(groups["GrB typedef enums"]) - set(groups["GxB typedef enums"])
     assert not missing_enums
 
     vals = {x for x in lines if "typedef" in x and "GxB" in x} - seen
@@ -757,9 +749,7 @@ def main():
 
     # Run it through the preprocessor
     print(f"Step 2: run preprocessor to create {processed_h}")
-    include = os.path.join(
-        os.path.dirname(pycparser.__file__), "utils", "fake_libc_include"
-    )
+    include = os.path.join(os.path.dirname(pycparser.__file__), "utils", "fake_libc_include")
     command = (
         f"gcc -nostdinc -E -I{include} {graphblas_h} "
         f"| sed 's/ complex / _Complex /g' > {processed_h}"
@@ -804,15 +794,12 @@ def main():
     unknown_defines = defines - DEFINES - CHAR_DEFINES - IGNORE_DEFINES
     if unknown_defines:
         raise ValueError(
-            f"Unknown #define values found in {graphblas_h}: "
-            + ", ".join(sorted(unknown_defines))
+            f"Unknown #define values found in {graphblas_h}: " + ", ".join(sorted(unknown_defines))
         )
     print("Success!", "\N{ROCKET}")
     if args.show_skipped:
         print()
-        print(
-            f"Showing lines from {processed_h} that were skipped when creating {final_h}:"
-        )
+        print(f"Showing lines from {processed_h} that were skipped when creating {final_h}:")
         print("-" * 80)
         for line in sorted(groups["not seen"], key=sort_key):
             print(line)

suitesparse_graphblas/create_headers.py

@@ -1,4 +1,5 @@
 from suitesparse_graphblas import lib, ffi
 
+
 def test_matrix_existence():
     assert hasattr(lib, "GrB_Matrix_new")

suitesparse_graphblas/tests/__init__.py

@@ -0,0 +1,34 @@
+from numpy cimport ndarray
+from libc.stdint cimport uint64_t
+
+cdef extern from "numpy/arrayobject.h" nogil:
+    # These aren't public (i.e., "extern"), but other projects use them too
+    void *PyDataMem_NEW(size_t)
+    void *PyDataMem_NEW_ZEROED(size_t, size_t)
+    void *PyDataMem_RENEW(void *, size_t)
+    void PyDataMem_FREE(void *)
+    # These are available in newer Cython versions
+    void PyArray_ENABLEFLAGS(ndarray, int flags)
+    void PyArray_CLEARFLAGS(ndarray, int flags)
+
+ctypedef enum GrB_Mode:
+    GrB_NONBLOCKING
+    GrB_BLOCKING
+
+ctypedef uint64_t (*GxB_init)(
+    GrB_Mode,
+    void *(*user_malloc_function)(size_t),
+    void *(*user_calloc_function)(size_t, size_t),
+    void *(*user_realloc_function)(void *, size_t),
+    void (*user_free_function)(void *),
+    bint,  # user_malloc_is_thread_safe
+)
+
+cpdef int call_gxb_init(ffi, lib, int mode)
+
+cpdef ndarray claim_buffer(ffi, cdata, size_t size, dtype)
+
+cpdef ndarray claim_buffer_2d(ffi, cdata, size_t cdata_size, size_t nrows, size_t ncols, dtype, bint is_c_order)
+
+cpdef unclaim_buffer(ndarray array)
+

tests/test_package.py renamed to suitesparse_graphblas/tests/test_package.py

@@ -0,0 +1,70 @@
+import numpy as np
+from numpy cimport (
+    import_array, ndarray, npy_intp,
+    PyArray_SimpleNewFromData, PyArray_New,
+    NPY_ARRAY_OWNDATA, NPY_ARRAY_WRITEABLE, NPY_ARRAY_F_CONTIGUOUS,
+)
+from libc.stdint cimport uintptr_t
+
+import_array()
+
+cpdef int call_gxb_init(ffi, lib, int mode):
+    # We need to call `GxB_init`, but we didn't compile Cython against GraphBLAS.  So, we get it from cffi.
+    # Step 1: ffi.addressof(lib, "GxB_init")
+    #    Return type: cffi.cdata object of a function pointer.  Can't cast to int.
+    # Step 2: ffi.cast("uintptr_t", ...)
+    #    Return type: cffi.cdata object of a uintptr_t type, an unsigned pointer.  Can cast to int.
+    # Step 3: int(...)
+    #    Return type: int.  The physical address of the function.
+    # Step 4: <uintptr_t>(...)
+    #    Return type: uintptr_t in Cython.  Cast Python int to Cython integer for pointers.
+    # Step 5: <GsB_init>(...)
+    #    Return: function pointer in Cython!
+
+    cdef GxB_init func = <GxB_init><uintptr_t>int(ffi.cast("uintptr_t", ffi.addressof(lib, "GxB_init")))
+    return func(<GrB_Mode>mode, PyDataMem_NEW, PyDataMem_NEW_ZEROED, PyDataMem_RENEW, PyDataMem_FREE, True)
+
+
+cpdef ndarray claim_buffer(ffi, cdata, size_t size, dtype):
+    cdef:
+        npy_intp dims = size
+        uintptr_t ptr = int(ffi.cast("uintptr_t", cdata))
+        ndarray array = PyArray_SimpleNewFromData(1, &dims, dtype.num, <void*>ptr)
+    PyArray_ENABLEFLAGS(array, NPY_ARRAY_OWNDATA)
+    return array
+
+
+cpdef ndarray claim_buffer_2d(ffi, cdata, size_t cdata_size, size_t nrows, size_t ncols, dtype, bint is_c_order):
+    cdef:
+        size_t size = nrows * ncols
+        ndarray array
+        uintptr_t ptr
+        npy_intp dims[2]
+    if cdata_size == size:
+        ptr = int(ffi.cast("uintptr_t", cdata))
+        dims[0] = nrows
+        dims[1] = ncols
+        if is_c_order:
+            array = PyArray_SimpleNewFromData(2, dims, dtype.num, <void*>ptr)
+        else:
+            array = PyArray_New(
+                ndarray, 2, dims, dtype.num, NULL, <void*>ptr, -1,
+                NPY_ARRAY_F_CONTIGUOUS | NPY_ARRAY_WRITEABLE, <object>NULL
+            )
+        PyArray_ENABLEFLAGS(array, NPY_ARRAY_OWNDATA)
+    elif cdata_size > size:  # pragma: no cover
+        array = claim_buffer(ffi, cdata, cdata_size, dtype)
+        if is_c_order:
+            array = array[:size].reshape((nrows, ncols))
+        else:
+            array = array[:size].reshape((ncols, nrows)).T
+    else:  # pragma: no cover
+        raise ValueError(
+            f"Buffer size too small: {cdata_size}.  "
+            f"Unable to create matrix of size {nrows}x{ncols} = {size}"
+        )
+    return array
+
+
+cpdef unclaim_buffer(ndarray array):
+    PyArray_CLEARFLAGS(array, NPY_ARRAY_OWNDATA | NPY_ARRAY_WRITEABLE)