[libc][mathvec] Initial commit for LIBC vector math component

libc/CMakeLists.txt

@@ -350,6 +350,7 @@ include(LLVMLibCRules)
 set(TARGET_LLVMLIBC_ENTRYPOINTS "")
 set(TARGET_LIBC_ENTRYPOINTS "")
 set(TARGET_LIBM_ENTRYPOINTS "")
+set(TARGET_LIBMVEC_ENTRYPOINTS "")
 set(TARGET_LLVMLIBC_REMOVED_ENTRYPOINTS "")
 
 # Check entrypoints.txt
@@ -380,6 +381,7 @@ foreach(removed_entrypoint IN LISTS TARGET_LLVMLIBC_REMOVED_ENTRYPOINTS)
   list(REMOVE_ITEM TARGET_LLVMLIBC_ENTRYPOINTS ${removed_entrypoint})
   list(REMOVE_ITEM TARGET_LIBC_ENTRYPOINTS ${removed_entrypoint})
   list(REMOVE_ITEM TARGET_LIBM_ENTRYPOINTS ${removed_entrypoint})
+  list(REMOVE_ITEM TARGET_LIBMVEC_ENTRYPOINTS ${removed_entrypoint})
 endforeach()
 
 set(TARGET_ENTRYPOINT_NAME_LIST "")

libc/config/linux/aarch64/entrypoints.txt

@@ -1178,7 +1178,14 @@ if(LLVM_LIBC_FULL_BUILD)
   )
 endif()
 
+if(LIBC_COMPILER_HAS_EXT_VECTOR_TYPE)
+  set(TARGET_LIBMVEC_ENTRYPOINTS
+    libc.src.mathvec.expf
+  )
+endif()
+
 set(TARGET_LLVMLIBC_ENTRYPOINTS
   ${TARGET_LIBC_ENTRYPOINTS}
   ${TARGET_LIBM_ENTRYPOINTS}
+  ${TARGET_LIBMVEC_ENTRYPOINTS}
 )

libc/config/linux/x86_64/entrypoints.txt

@@ -1396,7 +1396,14 @@ if(LLVM_LIBC_FULL_BUILD)
   )
 endif()
 
+if(LIBC_COMPILER_HAS_EXT_VECTOR_TYPE)
+  set(TARGET_LIBMVEC_ENTRYPOINTS
+    libc.src.mathvec.expf
+  )
+endif()
+
 set(TARGET_LLVMLIBC_ENTRYPOINTS
   ${TARGET_LIBC_ENTRYPOINTS}
   ${TARGET_LIBM_ENTRYPOINTS}
+  ${TARGET_LIBMVEC_ENTRYPOINTS}
 )

libc/lib/CMakeLists.txt

@@ -2,10 +2,10 @@ set(libc_archive_targets "")
 set(libc_archive_names "")
 set(libc_archive_entrypoint_lists "")
 if(LLVM_LIBC_FULL_BUILD)
-  list(APPEND libc_archive_names c m)
-  list(APPEND libc_archive_targets libc libm)
+  list(APPEND libc_archive_names c m mvec)
+  list(APPEND libc_archive_targets libc libm libmvec)
   list(APPEND libc_archive_entrypoint_lists
-       TARGET_LIBC_ENTRYPOINTS TARGET_LIBM_ENTRYPOINTS)
+       TARGET_LIBC_ENTRYPOINTS TARGET_LIBM_ENTRYPOINTS TARGET_LIBMVEC_ENTRYPOINTS)
 else()
   list(APPEND libc_archive_names llvmlibc)
   list(APPEND libc_archive_targets libc)

libc/src/CMakeLists.txt

@@ -29,6 +29,10 @@ if(${LIBC_TARGET_OS} STREQUAL "linux")
   add_subdirectory(termios)
 endif()
 
+if(LIBC_COMPILER_HAS_EXT_VECTOR_TYPE)
+  add_subdirectory(mathvec)
+endif()
+
 if(NOT LLVM_LIBC_FULL_BUILD)
   return()
 endif()

libc/src/__support/CMakeLists.txt

@@ -416,3 +416,6 @@ if(NOT (LIBC_TARGET_OS_IS_DARWIN))
 endif()
 
 add_subdirectory(math)
+if(LIBC_COMPILER_HAS_EXT_VECTOR_TYPE)
+  add_subdirectory(mathvec)
+endif()

libc/src/__support/CPP/bit.h

@@ -39,7 +39,7 @@ LIBC_INLINE static void inline_copy(const char *from, char *to) {
 // This implementation of bit_cast requires trivially-constructible To, to avoid
 // UB in the implementation.
 template <typename To, typename From>
-LIBC_INLINE constexpr cpp::enable_if_t<
+LIBC_INLINE static constexpr cpp::enable_if_t<
     (sizeof(To) == sizeof(From)) &&
         cpp::is_trivially_constructible<To>::value &&
         cpp::is_trivially_copyable<To>::value &&

libc/src/__support/macros/properties/cpu_features.h

@@ -56,6 +56,10 @@
 #define LIBC_TARGET_CPU_HAS_AVX512BW
 #endif
 
+#if defined(__AVX512F__) || defined(__AVX2__)
+#define LIBC_TARGET_CPU_HAS_GATHER
+#endif
+
 #if defined(__ARM_FP)
 #if (__ARM_FP & 0x2)
 #define LIBC_TARGET_CPU_HAS_ARM_FPU_HALF

libc/src/__support/mathvec/CMakeLists.txt

@@ -0,0 +1,27 @@
+add_header_library(
+  common_constants
+  HDRS
+    common_constants.h
+  DEPENDS
+)
+
+add_header_library(
+  expf_utils
+  HDRS
+    expf_utils.h
+  DEPENDS
+  libc.src.__support.CPP.simd
+  libc.src.__support.mathvec.common_constants
+)
+
+add_header_library(
+  expf
+  HDRS
+    expf.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.CPP.simd
+    libc.src.__support.FPUtil.FPBits
+    libc.src.__support.mathvec.expf_utils
+    libc.src.__support.mathvec.vector_utils
+)

libc/src/__support/mathvec/common_constants.h

@@ -0,0 +1,40 @@
+//===-- Common constants for mathvec functions ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATHVEC_COMMON_CONSTANTS_H
+#define LLVM_LIBC_SRC___SUPPORT_MATHVEC_COMMON_CONSTANTS_H
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace common_constants_internal {
+
+// Lookup table for mantissas of 2^(i / 64) with i = 0, ..., 63.
+static constexpr uint64_t EXP_MANTISSA[64] = {
+    0x0000000000000, 0x02c9a3e778061, 0x059b0d3158574, 0x0874518759bc8,
+    0x0b5586cf9890f, 0x0e3ec32d3d1a2, 0x11301d0125b51, 0x1429aaea92de0,
+    0x172b83c7d517b, 0x1a35beb6fcb75, 0x1d4873168b9aa, 0x2063b88628cd6,
+    0x2387a6e756238, 0x26b4565e27cdd, 0x29e9df51fdee1, 0x2d285a6e4030b,
+    0x306fe0a31b715, 0x33c08b26416ff, 0x371a7373aa9cb, 0x3a7db34e59ff7,
+    0x3dea64c123422, 0x4160a21f72e2a, 0x44e086061892d, 0x486a2b5c13cd0,
+    0x4bfdad5362a27, 0x4f9b2769d2ca7, 0x5342b569d4f82, 0x56f4736b527da,
+    0x5ab07dd485429, 0x5e76f15ad2148, 0x6247eb03a5585, 0x6623882552225,
+    0x6a09e667f3bcd, 0x6dfb23c651a2f, 0x71f75e8ec5f74, 0x75feb564267c9,
+    0x7a11473eb0187, 0x7e2f336cf4e62, 0x82589994cce13, 0x868d99b4492ed,
+    0x8ace5422aa0db, 0x8f1ae99157736, 0x93737b0cdc5e5, 0x97d829fde4e50,
+    0x9c49182a3f090, 0xa0c667b5de565, 0xa5503b23e255d, 0xa9e6b5579fdbf,
+    0xae89f995ad3ad, 0xb33a2b84f15fb, 0xb7f76f2fb5e47, 0xbcc1e904bc1d2,
+    0xc199bdd85529c, 0xc67f12e57d14b, 0xcb720dcef9069, 0xd072d4a07897c,
+    0xd5818dcfba487, 0xda9e603db3285, 0xdfc97337b9b5f, 0xe502ee78b3ff6,
+    0xea4afa2a490da, 0xefa1bee615a27, 0xf50765b6e4540, 0xfa7c1819e90d8,
+};
+
+} // namespace common_constants_internal
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATHVEC_COMMON_CONSTANTS_H

libc/src/__support/mathvec/expf.h

@@ -0,0 +1,84 @@
+//===-- Implementation header for SIMD expf ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATHVEC_EXPF_H
+#define LLVM_LIBC_SRC___SUPPORT_MATHVEC_EXPF_H
+
+#include "src/__support/CPP/simd.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/common.h"
+#include "src/__support/mathvec/expf_utils.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace mathvec {
+
+template <size_t N>
+LIBC_INLINE static cpp::simd<double, N> inline_exp(cpp::simd<double, N> x) {
+  static constexpr cpp::simd<double, N> shift = 0x1.800000000ffc0p+46;
+
+  // inv_ln2 = round(1/log(2), D, RN);
+  static constexpr cpp::simd<double, N> inv_ln2 = 0x1.71547652b82fep+0;
+  cpp::simd<double, N> z = shift + x * inv_ln2;
+  cpp::simd<double, N> n = z - shift;
+
+  // ln2_hi = round(log(2), D, RN);
+  // ln2_lo = round(log(2) - ln2_hi, D, RN);
+  static constexpr cpp::simd<double, N> ln2_hi = 0x1.62e42fefa39efp-1;
+  static constexpr cpp::simd<double, N> ln2_lo = 0x1.abc9e3b39803fp-56;
+
+  cpp::simd<double, N> r = x;
+  r = r - n * ln2_hi;
+  r = r - n * ln2_lo;
+
+  // Coefficients of exp approximation, generated by Sollya with:
+  // poly = 1 + x;
+  // for i from 2 to 5 do {
+  //   r = remez(exp(x)-poly(x), 5-i, [-log(2)/128;log(2)/128], x^i, 1e-10);
+  //   c = coeff(roundcoefficients(r, [|D ...|]), 0);
+  //   poly = poly + x^i*c;
+  //   c;
+  // };
+  static constexpr cpp::simd<double, N> c0 = 0x1.fffffffffdbcep-2;
+  static constexpr cpp::simd<double, N> c1 = 0x1.55555555543c2p-3;
+  static constexpr cpp::simd<double, N> c2 = 0x1.555573c64f2e3p-5;
+  static constexpr cpp::simd<double, N> c3 = 0x1.111126b4eff73p-7;
+
+  /* y = exp(r) - 1 ~= r + C0 r^2 + C1 r^3 + C2 r^4 + C3 r^5.  */
+  cpp::simd<double, N> r2 = r * r;
+  cpp::simd<double, N> p01 = c0 + r * c1;
+  cpp::simd<double, N> p23 = c2 + r * c3;
+  cpp::simd<double, N> p04 = p01 + r2 * p23;
+  cpp::simd<double, N> y = r + p04 * r2;
+
+  cpp::simd<uint64_t, N> u = cpp::bit_cast<cpp::simd<uint64_t, N>>(z);
+  cpp::simd<double, N> s = exp_lookup(u);
+  return s + s * y;
+}
+
+template <size_t N>
+LIBC_INLINE cpp::simd<float, N> expf(cpp::simd<float, N> x) {
+  using FPBits = typename fputil::FPBits<float>;
+
+  cpp::simd<bool, N> is_inf = x >= 0x1.62e38p+9;
+  cpp::simd<bool, N> is_zero = x <= -0x1.628c2ap+9;
+  cpp::simd<bool, N> is_special = is_inf | is_zero;
+
+  cpp::simd<float, N> special_res = is_inf ? FPBits::inf().get_val() : 0.0f;
+
+  cpp::simd<double, N> x_d = cpp::simd_cast<double, float, N>(x);
+  cpp::simd<double, N> y = inline_exp(x_d);
+  cpp::simd<float, N> ret = cpp::simd_cast<float, double, N>(y);
+  return is_special ? special_res : ret;
+}
+
+} // namespace mathvec
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATHVEC_EXPF_H

libc/src/__support/mathvec/expf_utils.h

@@ -0,0 +1,29 @@
+//===-- Common utils for exp function ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATHVEC_EXP_UTILS_H
+#define LLVM_LIBC_SRC___SUPPORT_MATHVEC_EXP_UTILS_H
+
+#include "src/__support/CPP/simd.h"
+#include "src/__support/mathvec/common_constants.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+template <size_t N>
+LIBC_INLINE static cpp::simd<double, N> exp_lookup(cpp::simd<uint64_t, N> u) {
+  cpp::simd<uint64_t, N> index = u & cpp::simd<uint64_t, N>(0x3f);
+  cpp::simd<uint64_t, N> mantissa = cpp::gather<cpp::simd<uint64_t, N>>(
+      true, index, common_constants_internal::EXP_MANTISSA);
+  cpp::simd<uint64_t, N> exponent = (u >> 6) << 52;
+  cpp::simd<uint64_t, N> result = mantissa | exponent;
+  return cpp::bit_cast<cpp::simd<double, N>>(result);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATHVEC_EXP_UTILS_H

libc/src/mathvec/CMakeLists.txt

@@ -0,0 +1,43 @@
+add_subdirectory(generic)
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
+  add_subdirectory(${LIBC_TARGET_ARCHITECTURE})
+endif()
+
+function(add_vector_math_entrypoint_object name)
+  # We prefer machine specific implementation if available. Hence we check
+  # that first and return early if we are able to add an alias target for the
+  # machine specific implementation.
+  get_fq_target_name("${LIBC_TARGET_ARCHITECTURE}.${name}" fq_machine_specific_target_name)
+  if(TARGET ${fq_machine_specific_target_name})
+    add_entrypoint_object(
+      ${name}
+      ALIAS
+      DEPENDS
+        .${LIBC_TARGET_ARCHITECTURE}.${name}
+    )
+    return()
+  endif()
+
+  get_fq_target_name("generic.${name}" fq_generic_target_name)
+  if(TARGET ${fq_generic_target_name})
+    add_entrypoint_object(
+      ${name}
+      ALIAS
+      DEPENDS
+        .generic.${name}
+    )
+    return()
+  endif()
+
+  # Add a dummy entrypoint object for missing implementations. They will be skipped
+  # anyway as there will be no entry for them in the target entrypoints list.
+  add_entrypoint_object(
+    ${name}
+    SRCS
+      dummy_srcs
+    HDRS
+      dummy_hdrs
+  )
+endfunction()
+
+add_vector_math_entrypoint_object(expf)

libc/src/mathvec/expf.h

@@ -0,0 +1,21 @@
+//===-- Implementation header for SIMD expf ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATHVEC_EXPF_H
+#define LLVM_LIBC_SRC_MATHVEC_EXPF_H
+
+#include "src/__support/CPP/simd.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+cpp::simd<float> expf(cpp::simd<float> x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATHVEC_EXPF_H

libc/src/mathvec/generic/CMakeLists.txt

@@ -0,0 +1,12 @@
+add_entrypoint_object(
+  expf
+  SRCS
+    expf.cpp
+  HDRS
+    ../expf.h
+  DEPENDS
+    libc.src.__support.CPP.simd
+  FLAGS
+    ROUND_OPT
+    FMA_OPT
+)

libc/src/mathvec/generic/expf.cpp

@@ -0,0 +1,18 @@
+//===-- Single-precision SIMD e^x vector function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/mathvec/expf.h"
+#include "src/__support/mathvec/expf.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(cpp::simd<float>, expf, (cpp::simd<float> x)) {
+  return mathvec::expf(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL