Merge branch 'master' into devel

Author: pavanky

arrayfire/interop.py

@@ -10,44 +10,88 @@
 """
 Interop with other python packages.
 
-This module provides interoperability with the following python packages.
+This module provides helper functions to copy data to arrayfire from the following modules:
+
+     1. numpy - numpy.ndarray
+     2. pycuda - pycuda.gpuarray
+     3. pyopencl - pyopencl.array
+     4. numba - numba.cuda.cudadrv.devicearray.DeviceNDArray
 
-     1. numpy
-     2. pycuda
-     3. pyopencl
 """
 
 from .array import *
 from .device import *
 
+
+def _fc_to_af_array(in_ptr, in_shape, in_dtype, is_device=False, copy = True):
+    """
+    Fortran Contiguous to af array
+    """
+    res = Array(in_ptr, in_shape, in_dtype, is_device=is_device)
+
+    if is_device:
+        lock_array(res)
+        pass
+
+    return res.copy() if copy else res
+
+def _cc_to_af_array(in_ptr, ndim, in_shape, in_dtype, is_device=False, copy = True):
+    """
+    C Contiguous to af array
+    """
+    if ndim == 1:
+        return _fc_to_af_array(in_ptr, in_shape, in_dtype, is_device, copy)
+    elif ndim == 2:
+        shape = (in_shape[1], in_shape[0])
+        res = Array(in_ptr, shape, in_dtype, is_device=is_device)
+        if is_device: lock_array(res)
+        return reorder(res, 1, 0)
+    elif ndim == 3:
+        shape = (in_shape[2], in_shape[1], in_shape[0])
+        res = Array(in_ptr, shape, in_dtype, is_device=is_device)
+        if is_device: lock_array(res)
+        return reorder(res, 2, 1, 0)
+    elif ndim == 4:
+        shape = (in_shape[3], in_shape[2], in_shape[1], in_shape[0])
+        res = Array(in_ptr, shape, in_dtype, is_device=is_device)
+        if is_device: lock_array(res)
+        return reorder(res, 3, 2, 1, 0)
+    else:
+        raise RuntimeError("Unsupported ndim")
+
+
+_nptype_to_aftype = {'b1' : Dtype.b8,
+		     'u1' : Dtype.u8,
+		     'u2' : Dtype.u16,
+		     'i2' : Dtype.s16,
+		     's4' : Dtype.u32,
+		     'i4' : Dtype.s32,
+		     'f4' : Dtype.f32,
+		     'c8' : Dtype.c32,
+		     's8' : Dtype.u64,
+		     'i8' : Dtype.s64,
+                     'f8' : Dtype.f64,
+                     'c16' : Dtype.c64}
+
 try:
     import numpy as np
     from numpy import ndarray as NumpyArray
     from .data import reorder
 
     AF_NUMPY_FOUND=True
 
-    _nptype_to_aftype = {'b1' : Dtype.b8,
-			 'u1' : Dtype.u8,
-			 'u2' : Dtype.u16,
-			 'i2' : Dtype.s16,
-			 's4' : Dtype.u32,
-			 'i4' : Dtype.s32,
-			 'f4' : Dtype.f32,
-			 'c8' : Dtype.c32,
-			 's8' : Dtype.u64,
-			 'i8' : Dtype.s64,
-                         'f8' : Dtype.f64,
-                         'c16' : Dtype.c64}
-
-    def np_to_af_array(np_arr):
+    def np_to_af_array(np_arr, copy=True):
         """
         Convert numpy.ndarray to arrayfire.Array.
 
         Parameters
         ----------
         np_arr  : numpy.ndarray()
 
+        copy : Bool specifying if array is to be copied.
+               Default is true.
+               Can only be False if array is fortran contiguous.
+
         Returns
         ---------
         af_arr  : arrayfire.Array()
@@ -57,27 +101,15 @@ def np_to_af_array(np_arr):
         in_ptr = np_arr.ctypes.data_as(c_void_ptr_t)
         in_dtype = _nptype_to_aftype[np_arr.dtype.str[1:]]
 
+        if not copy:
+            raise RuntimeError("Copy can not be False for numpy arrays")
+
         if (np_arr.flags['F_CONTIGUOUS']):
-            return Array(in_ptr, in_shape, in_dtype)
+            return _fc_to_af_array(in_ptr, in_shape, in_dtype)
         elif (np_arr.flags['C_CONTIGUOUS']):
-            if np_arr.ndim == 1:
-                return Array(in_ptr, in_shape, in_dtype)
-            elif np_arr.ndim == 2:
-                shape = (in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype)
-                return reorder(res, 1, 0)
-            elif np_arr.ndim == 3:
-                shape = (in_shape[2], in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype)
-                return reorder(res, 2, 1, 0)
-            elif np_arr.ndim == 4:
-                shape = (in_shape[3], in_shape[2], in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype)
-                return reorder(res, 3, 2, 1, 0)
-            else:
-                raise RuntimeError("Unsupported ndim")
+            return _cc_to_af_array(in_ptr, np_arr.ndim, in_shape, in_dtype)
         else:
-            return np_to_af_array(np.asfortranarray(np_arr))
+            return np_to_af_array(np_arr.copy())
 
     from_ndarray = np_to_af_array
 except:
@@ -88,14 +120,18 @@ def np_to_af_array(np_arr):
     from pycuda.gpuarray import GPUArray as CudaArray
     AF_PYCUDA_FOUND=True
 
-    def pycuda_to_af_array(pycu_arr):
+    def pycuda_to_af_array(pycu_arr, copy=True):
         """
         Convert pycuda.gpuarray to arrayfire.Array
 
         Parameters
         -----------
         pycu_arr  : pycuda.GPUArray()
 
+        copy : Bool specifying if array is to be copied.
+               Default is true.
+               Can only be False if array is fortran contiguous.
+
         Returns
         ----------
         af_arr    : arrayfire.Array()
@@ -109,31 +145,13 @@ def pycuda_to_af_array(pycu_arr):
         in_shape = pycu_arr.shape
         in_dtype = pycu_arr.dtype.char
 
+        if not copy and not pycu_arr.flags.f_contiguous:
+            raise RuntimeError("Copy can only be False when arr.flags.f_contiguous is True")
+
         if (pycu_arr.flags.f_contiguous):
-            res = Array(in_ptr, in_shape, in_dtype, is_device=True)
-            lock_array(res)
-            res = res.copy()
-            return res
+            return _fc_to_af_array(in_ptr, in_shape, in_dtype, True, copy)
         elif (pycu_arr.flags.c_contiguous):
-            if pycu_arr.ndim == 1:
-                return Array(in_ptr, in_shape, in_dtype, is_device=True)
-            elif pycu_arr.ndim == 2:
-                shape = (in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype, is_device=True)
-                lock_array(res)
-                return reorder(res, 1, 0)
-            elif pycu_arr.ndim == 3:
-                shape = (in_shape[2], in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype, is_device=True)
-                lock_array(res)
-                return reorder(res, 2, 1, 0)
-            elif pycu_arr.ndim == 4:
-                shape = (in_shape[3], in_shape[2], in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype, is_device=True)
-                lock_array(res)
-                return reorder(res, 3, 2, 1, 0)
-            else:
-                raise RuntimeError("Unsupported ndim")
+            return _cc_to_af_array(in_ptr, pycu_arr.ndim, in_shape, in_dtype, True, copy)
         else:
             return pycuda_to_af_array(pycu_arr.copy())
 except:
@@ -147,14 +165,18 @@ def pycuda_to_af_array(pycu_arr):
     from .opencl import get_context as _get_context
     AF_PYOPENCL_FOUND=True
 
-    def pyopencl_to_af_array(pycl_arr):
+    def pyopencl_to_af_array(pycl_arr, copy=True):
         """
         Convert pyopencl.gpuarray to arrayfire.Array
 
         Parameters
         -----------
         pycl_arr  : pyopencl.Array()
 
+        copy : Bool specifying if array is to be copied.
+               Default is true.
+               Can only be False if array is fortran contiguous.
+
         Returns
         ----------
         af_arr    : arrayfire.Array()
@@ -179,63 +201,102 @@ def pyopencl_to_af_array(pycl_arr):
 
         if (dev_idx == None or ctx_idx == None or
             dev_idx != dev or ctx_idx != ctx):
+            print("Adding context and queue")
             _add_device_context(dev, ctx, que)
             _set_device_context(dev, ctx)
 
+        info()
         in_ptr = pycl_arr.base_data.int_ptr
         in_shape = pycl_arr.shape
         in_dtype = pycl_arr.dtype.char
 
+        if not copy and not pycl_arr.flags.f_contiguous:
+            raise RuntimeError("Copy can only be False when arr.flags.f_contiguous is True")
+
+        print("Copying array")
+        print(pycl_arr.base_data.int_ptr)
         if (pycl_arr.flags.f_contiguous):
-            res = Array(in_ptr, in_shape, in_dtype, is_device=True)
-            lock_array(res)
-            return res
+            return _fc_to_af_array(in_ptr, in_shape, in_dtype, True, copy)
         elif (pycl_arr.flags.c_contiguous):
-            if pycl_arr.ndim == 1:
-                return Array(in_ptr, in_shape, in_dtype, is_device=True)
-            elif pycl_arr.ndim == 2:
-                shape = (in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype, is_device=True)
-                lock_array(res)
-                return reorder(res, 1, 0)
-            elif pycl_arr.ndim == 3:
-                shape = (in_shape[2], in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype, is_device=True)
-                lock_array(res)
-                return reorder(res, 2, 1, 0)
-            elif pycl_arr.ndim == 4:
-                shape = (in_shape[3], in_shape[2], in_shape[1], in_shape[0])
-                res = Array(in_ptr, shape, in_dtype, is_device=True)
-                lock_array(res)
-                return reorder(res, 3, 2, 1, 0)
-            else:
-                raise RuntimeError("Unsupported ndim")
+            return _cc_to_af_array(in_ptr, pycl_arr.ndim, in_shape, in_dtype, True, copy)
         else:
             return pyopencl_to_af_array(pycl_arr.copy())
 except:
     AF_PYOPENCL_FOUND=False
 
+try:
+    import numba
+    from numba import cuda
+    NumbaCudaArray = cuda.cudadrv.devicearray.DeviceNDArray
+    AF_NUMBA_FOUND=True
+
+    def numba_to_af_array(nb_arr, copy=True):
+        """
+        Convert numba.gpuarray to arrayfire.Array
+
+        Parameters
+        -----------
+        nb_arr  : numba.cuda.cudadrv.devicearray.DeviceNDArray()
+
+        copy : Bool specifying if array is to be copied.
+               Default is true.
+               Can only be False if array is fortran contiguous.
+
+        Returns
+        ----------
+        af_arr    : arrayfire.Array()
 
-def to_array(in_array):
+        Note
+        ----------
+        The input array is copied to af.Array
+        """
+
+        in_ptr = nb_arr.device_ctypes_pointer.value
+        in_shape = nb_arr.shape
+        in_dtype = _nptype_to_aftype[nb_arr.dtype.str[1:]]
+
+        if not copy and not nb_arr.flags.f_contiguous:
+            raise RuntimeError("Copy can only be False when arr.flags.f_contiguous is True")
+
+        if (nb_arr.is_f_contiguous()):
+            return _fc_to_af_array(in_ptr, in_shape, in_dtype, True, copy)
+        elif (nb_arr.is_c_contiguous()):
+            return _cc_to_af_array(in_ptr, nb_arr.ndim, in_shape, in_dtype, True, copy)
+        else:
+            return numba_to_af_array(nb_arr.copy())
+except:
+    AF_NUMBA_FOUND=False
+
+def to_array(in_array, copy = True):
     """
     Helper function to convert input from a different module to af.Array
 
     Parameters
     -------------
 
     in_array : array like object
-             Can be one of numpy.ndarray, pycuda.GPUArray, pyopencl.Array, array.array, list
+             Can be one of the following:
+             - numpy.ndarray
+             - pycuda.GPUArray
+             - pyopencl.Array
+             - numba.cuda.cudadrv.devicearray.DeviceNDArray
+             - array.array
+             - list
+    copy : Bool specifying if array is to be copied.
+          Default is true.
+          Can only be False if array is fortran contiguous.
 
     Returns
     --------------
     af.Array of same dimensions as input after copying the data from the input
 
-
     """
     if AF_NUMPY_FOUND and isinstance(in_array, NumpyArray):
-        return np_to_af_array(in_array)
+        return np_to_af_array(in_array, copy)
     if AF_PYCUDA_FOUND and isinstance(in_array, CudaArray):
-        return pycuda_to_af_array(in_array)
+        return pycuda_to_af_array(in_array, copy)
     if AF_PYOPENCL_FOUND and isinstance(in_array, OpenclArray):
-        return pyopencl_to_af_array(in_array)
+        return pyopencl_to_af_array(in_array, copy)
+    if AF_NUMBA_FOUND and isinstance(in_array, NumbaCudaArray):
+        return numba_to_af_array(in_array, copy)
     return Array(src=in_array)

arrayfire/opencl.py

@@ -54,7 +54,7 @@ def get_context(retain=False):
 
     import ctypes as ct
     from .util import safe_call as safe_call
-    from .library import backend as backend
+    from .library import backend
 
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
@@ -80,7 +80,7 @@ def get_queue(retain):
 
     import ctypes as ct
     from .util import safe_call as safe_call
-    from .library import backend as backend
+    from .library import backend
 
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
@@ -102,7 +102,7 @@ def get_device_id():
 
     import ctypes as ct
     from .util import safe_call as safe_call
-    from .library import backend as backend
+    from .library import backend
 
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
@@ -124,7 +124,7 @@ def set_device_id(idx):
 
     import ctypes as ct
     from .util import safe_call as safe_call
-    from .library import backend as backend
+    from .library import backend
 
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
@@ -146,6 +146,10 @@ def add_device_context(dev, ctx, que):
     que : cl_command_queue
 
     """
+    import ctypes as ct
+    from .util import safe_call as safe_call
+    from .library import backend
+
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
 
@@ -163,6 +167,10 @@ def set_device_context(dev, ctx):
     ctx  : cl_context
 
     """
+    import ctypes as ct
+    from .util import safe_call as safe_call
+    from .library import backend
+
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
 
@@ -180,6 +188,10 @@ def delete_device_context(dev, ctx):
     ctx  : cl_context
 
     """
+    import ctypes as ct
+    from .util import safe_call as safe_call
+    from .library import backend
+
     if (backend.name() != "opencl"):
         raise RuntimeError("Invalid backend loaded")
 
@@ -204,6 +216,13 @@ def get_device_type():
     """
     Get opencl device type
     """
+    import ctypes as ct
+    from .util import safe_call as safe_call
+    from .library import backend
+
+    if (backend.name() != "opencl"):
+        raise RuntimeError("Invalid backend loaded")
+
     res = c_int_t(DEVICE_TYPE.UNKNOWN.value)
     safe_call(backend.get().afcl_get_device_type(c_pointer(res)))
     return _to_device_type[res.value]
@@ -212,6 +231,13 @@ def get_platform():
     """
     Get opencl platform
     """
+    import ctypes as ct
+    from .util import safe_call as safe_call
+    from .library import backend
+
+    if (backend.name() != "opencl"):
+        raise RuntimeError("Invalid backend loaded")
+
     res = c_int_t(PLATFORM.UNKNOWN.value)
     safe_call(backend.get().afcl_get_platform(c_pointer(res)))
     return _to_platform[res.value]

arrayfire/tests/simple/__init__.py

@@ -15,6 +15,7 @@
 from .device import *
 from .image import *
 from .index import *
+from .interop import *
 from .lapack import *
 from .signal import *
 from .statistics import *

arrayfire/tests/simple/interop.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python
+#######################################################
+# Copyright (c) 2015, ArrayFire
+# All rights reserved.
+#
+# This file is distributed under 3-clause BSD license.
+# The complete license agreement can be obtained at:
+# http://arrayfire.com/licenses/BSD-3-Clause
+########################################################
+
+import arrayfire as af
+from . import _util
+
+def simple_interop(verbose = False):
+    if af.AF_NUMPY_FOUND:
+        import numpy as np
+        n = np.random.random((5,))
+        a = af.to_array(n)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3))
+        a = af.to_array(n)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3,2))
+        a = af.to_array(n)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3,2, 2))
+        a = af.to_array(n)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+    if af.AF_PYCUDA_FOUND and af.get_active_backend() == 'cuda':
+        import pycuda.autoinit
+        import pycuda.gpuarray as cudaArray
+        n = np.random.random((5,))
+        c = cudaArray.to_gpu(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3))
+        c = cudaArray.to_gpu(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3,2))
+        c = cudaArray.to_gpu(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3,2,2))
+        c = cudaArray.to_gpu(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+    if af.AF_PYOPENCL_FOUND and af.backend.name() == 'opencl':
+        # This needs fixing upstream
+        # https://github.com/arrayfire/arrayfire/issues/1728
+
+        # import pyopencl as cl
+        # import pyopencl.array as clArray
+        # ctx = cl.create_some_context()
+        # queue = cl.CommandQueue(ctx)
+
+        # n = np.random.random((5,))
+        # c = cl.array.to_device(queue, n)
+        # a = af.to_array(c)
+        # n2 = np.array(a)
+        # assert((n==n2).all())
+
+        # n = np.random.random((5,3))
+        # c = cl.array.to_device(queue, n)
+        # a = af.to_array(c)
+        # n2 = np.array(a)
+        # assert((n==n2).all())
+
+        # n = np.random.random((5,3,2))
+        # c = cl.array.to_device(queue, n)
+        # a = af.to_array(c)
+        # n2 = np.array(a)
+        # assert((n==n2).all())
+
+        # n = np.random.random((5,3,2,2))
+        # c = cl.array.to_device(queue, n)
+        # a = af.to_array(c)
+        # n2 = np.array(a)
+        # assert((n==n2).all())
+        pass
+
+    if af.AF_NUMBA_FOUND and af.get_active_backend() == 'cuda':
+
+        import numba
+        from numba import cuda
+
+        n = np.random.random((5,))
+        c = cuda.to_device(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3))
+        c = cuda.to_device(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3,2))
+        c = cuda.to_device(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+        n = np.random.random((5,3,2,2))
+        c = cuda.to_device(n)
+        a = af.to_array(c)
+        n2 = np.array(a)
+        assert((n==n2).all())
+
+_util.tests['interop'] = simple_interop