diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fd513e0d..91c1009db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,6 +456,26 @@ jobs: name: Run QA Automation (AILite) command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=light ./tests/qa/run + build-jetson-platform: + machine: + resource_class: arm.medium + image: ubuntu-2004:202101-01 + + steps: + - abort_for_docs + - abort_for_noci + - early_return_for_forked_pull_requests + - relocate-docker-storage + - checkout-all + - setup-automation + - run: + name: Build + command: | + pip3 install --user jinja2 + pushd opt/build/docker + docker login -u redisfab -p $DOCKER_REDISFAB_PWD + make REDIS_JETSON=1 PACK=0 TEST=0 VERBOSE=1 PUBLISH=1 + no_output_timeout: 40m #nightly-automation: # docker: @@ -559,6 +579,10 @@ workflows: - build-and-test-gpu-for-forked-prs: <<: *on-any-branch <<: *after-linter + - build-jetson-platform: + context: common + <<: *after-build-and-test + <<: *on-master-version-tags-and-dockertests - platforms-build: context: common <<: *after-build-and-test diff --git a/Dockerfile.arm b/Dockerfile.arm deleted file mode 100755 index 22e609e7f..000000000 --- a/Dockerfile.arm +++ /dev/null @@ -1,83 +0,0 @@ -# BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} - -ARG REDIS_VER=6.2.4 - -# OSNICK=bionic|stretch|buster -ARG OSNICK=buster - -# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic -ARG OS=debian:buster-slim - -# ARCH=arm64v8|arm32v7 -ARG ARCH=arm64v8 - -ARG PACK=0 -ARG REDISAI_LITE=0 -ARG TEST=0 - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis -FROM redisfab/xbuild:${ARCH}-${OS} AS builder - -ARG OSNICK -ARG OS -ARG ARCH -ARG REDIS_VER - -RUN [ "cross-build-start" ] - -RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH}" - -WORKDIR /build -COPY --from=redis /usr/local/ /usr/local/ - -COPY ./opt/ opt/ -COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow - -RUN ./opt/readies/bin/getpy3 -RUN ./opt/system-setup.py - -ARG DEPS_ARGS="" -COPY ./get_deps.sh . -RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi - -ARG BUILD_ARGS="" -ADD ./ /build -RUN set -e ;\ - . ./opt/readies/bin/sourced ./profile.d ;\ - make -C opt build $BUILD_ARGS SHOW=1 - -ARG PACK -ARG TEST - -RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi -RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi - -RUN [ "cross-build-end" ] - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK} - -RUN [ "cross-build-start" ] - -ARG OSNICK -ARG OS -ARG ARCH -ARG REDIS_VER -ARG PACK - -RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi -RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi - -ENV REDIS_MODULES /usr/lib/redis/modules -ENV LD_LIBRARY_PATH $REDIS_MODULES - -RUN mkdir -p $REDIS_MODULES/ - -COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ - -WORKDIR /data -EXPOSE 6379 -CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"] - -RUN [ "cross-build-end" ] diff --git a/Dockerfile.jetson b/Dockerfile.jetson deleted file mode 100644 index 34c544b71..000000000 --- a/Dockerfile.jetson +++ /dev/null @@ -1,102 +0,0 @@ -# BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK} - -ARG REDIS_VER=6.2.4 - -# OSNICK=bionic|centos7|centos6 -ARG OSNICK=bionic - -# OS=ubuntu18.04|ubuntu16.04|centos7 -ARG OS=ubuntu18.04 - -# ARCH=arm64v8|arm32v7 -ARG ARCH=arm64v8 - -ARG CUDA_VER=10.2-cudnn7 - -ARG L4T_VER=r32.4.4 - -ARG PACK=0 -ARG REDISAI_LITE=0 -ARG TEST=0 - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis -FROM redisfab/jetpack:4.4.1-arm64v8-l4t as builder - -ARG OSNICK -ARG OS -ARG ARCH -ARG REDIS_VER -ARG CUDA_VER -ARG L4T_VER - -RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]" - -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility - -WORKDIR /build -COPY --from=redis /usr/local/ /usr/local/ - -COPY ./opt/ opt/ -COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow/ - -RUN FORCE=1 ./opt/readies/bin/getpy3 -RUN ./opt/system-setup.py - -ARG DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0" -COPY ./get_deps.sh . -# RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi -RUN set -e ;\ - env $DEPS_ARGS ./get_deps.sh - -ARG BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0" - -ADD ./ /build -RUN bash -c "set -e ;\ - . ./opt/readies/bin/sourced ./profile.d ;\ - make -C opt build $BUILD_ARGS SHOW=1" - -ARG PACK -ARG REDISAI_LITE -ARG TEST - -RUN mkdir -p bin/artifacts -RUN set -e ;\ - if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi - -RUN set -e ;\ - if [ "$TEST" = "1" ]; then \ - TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\ - if [[ -d test/logs ]]; then \ - tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\ - fi ;\ - fi - -#---------------------------------------------------------------------------------------------- -# FROM nvidia/cuda:${CUDA_VER}-runtime-${OS} -# FROM nvidia/cuda-arm64:11.1-runtime-ubuntu18.04 -# FROM nvcr.io/nvidia/l4t-base:${L4T_VER} -FROM redisfab/jetpack:4.4.1-arm64v8-l4t - -ARG OS -ARG L4T_VER - -RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi -RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi - -ENV REDIS_MODULES /usr/lib/redis/modules -RUN mkdir -p $REDIS_MODULES/ - -COPY --from=redis /usr/local/ /usr/local/ -COPY --from=builder /build/install-gpu/ $REDIS_MODULES/ - -COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts - -# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/ - -WORKDIR /data -EXPOSE 6379 -# CMD ["/usr/local/bin/redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"] -RUN rm -rf /root/.cache /usr/local/cuda/lib64/*.a /usr/local/cuda/doc /usr/local/cuda/samples -CMD ["/usr/local/bin/redis-server", "--loadmodule", "/usr/lib/redis/modules/redisai.so"] diff --git a/README.md b/README.md index bfd4b54b5..357e86ab8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,13 @@ For docker instance with GPU support, you can launch it from `tensorwerk/redisai ```sh docker run -p 6379:6379 --gpus all -it --rm redislabs/redisai:edge-gpu + + +``` + +For a docker instance on an arm device we currently support CUDA 10.2 and CUDNN 7 - built on the nvidia jetson and xavier devices. The docker images below, are specific to those devices, and contain only the pytorch backend. +```sh +docker run -p 6379:6379 --gpus all -it --rm redislabs/redisai:edge-gpu-jetson ``` But if you'd like to build the docker image, you need a machine that has Nvidia driver (CUDA 10.0), nvidia-container-toolkit and Docker 19.03+ installed. For detailed information, checkout [nvidia-docker documentation](https://github.com/NVIDIA/nvidia-docker) diff --git a/get_deps.sh b/get_deps.sh index 11e18f8ae..fab45f890 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -5,6 +5,8 @@ error() { exit 1 } +set -x + trap error ERR HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" @@ -202,6 +204,9 @@ fi # WITH_TFLITE ###################################################################################### LIBTORCH PT_VERSION="1.7.1" +if [[ $JETSON == 1 ]]; then + PT_VERSION="1.7.0" +fi if [[ $WITH_PT != 0 ]]; then [[ $FORCE == 1 ]] && rm -rf $LIBTORCH diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index f0f1acaf2..ed3e48658 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -1,68 +1,43 @@ # Feel free to change these PRODUCT=redisai DOCKER_ORG=redislabs -REDIS_VERSION=6.2.4 -REDIS_CUDA_VERSION=11.0-cudnn8 -REDISAI_LITE=0 +export REDIS_VERSION=6.2.4 +export REDIS_CUDA_VERSION=11.0-cudnn8 +export REDISAI_LITE=0 PACK=1 # to fetch the artifacts - -DOCKER_OPTS= # set, to pass custom options to docker, but remember to quote them +TEST=1 # because dockers OSNICK ?= bionic -# set to pass multiple directories as docker sources to include in templates -DOCKER_SOURCES="" - -# set, to generate a file named Dockerfile<suffix>, i.e useful when you want to run things in parallel -DOCKER_SUFFIX= - -# Add space delimited docker tags, and they'll all be applied -DOCKER_TAGS= - DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-cpu-${OSNICK} +# jetson flavour +ifdef REDIS_JETSON +export REDIS_JETSON=1 +export REDIS_CUDA_VERSION=10.2-cudnn7 +DOCKER_SUFFIX=.jetson +DOCKER_ARGS=gpu_build=1 +DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-gpu-jetson +endif + # set gpu arguments ifeq ($(GPU),1) DOCKER_ARGS = gpu_build=1 -DOCKER_SUFFIX=gpu +DOCKER_SUFFIX=.gpu DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-gpu-${OSNICK} endif # remap ubuntu versions because of nvidia cuda ifeq ($(OSNICK),xenial) -REDIS_CUDA_UBUNTUVERSION=16.04 +export REDIS_CUDA_UBUNTUVERSION=16.04 endif ifeq ($(OSNICK),bionic) -REDIS_CUDA_UBUNTUVERSION=18.04 +export REDIS_CUDA_UBUNTUVERSION=18.04 endif -DOCKERWRAPPER_EXTRA_VARS=\ - REDISAI_LITE=${REDISAI_LITE} \ - REDIS_CUDA_VERSION=${REDIS_CUDA_VERSION} \ - REDIS_CUDA_UBUNTUVERSION=${REDIS_CUDA_UBUNTUVERSION} - ### Defaults ### ROOT=../../.. READIES=${ROOT}/opt/readies -#------------------------------------------------------------------------------ -define HELP -make build # build and optionally publish the docker, from a template file - OSNICK=<> # base operating system - REDISAI_LITE=1 # enable RedisAI lite builds - DOCKER_SUFFIX=<> # optional suffix for the generated dockerfile - DOCKER_ARGS=FOO=BAR # key-value pairs of variables to pass into the docker build - PACK=1 # fetch generated artifacts - DOCKER_TAGS=a,b,c # tags to append and push to dockerhub - DEFAULT_TAG=redislabs/redisai # default docker tag to build, and push - TEST=1 # run tests, if specified - VERSION=x.y.z # set the docker version - NOP=1 # set to echo files in docker generation, and not run - DOCKER_SOURCES=/a/path # append paths to the template generator - PUBLISH=1 # if set, push to dockerhub (requires docker login) - DOCKER_OPTS=XXX # Options to pass to the docker build command -endef -#------------------------------------------------------------------------------ - # rules, for building include ${READIES}/mk/docker.rules diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 261fd2863..e0fa14c40 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -1,23 +1,16 @@ -# BUILD redisfab/redisai:{{VERSION}}-cpu-{{REDIS_ARCH}}-{{REDIS_OSNICK}} - - -ARG PACK={{REDIS_PACK}} -ARG REDISAI_LITE={{REDISAI_LITE}} -ARG TEST={{REDIS_TEST}} - +# BUILD redisfab/redisai:{{REDIS_VERSION}}-cpu|gpu|jetson-{{REDIS_ARCH}}-{{REDIS_OSNICK}} #---------------------------------------------------------------------------------------------- FROM redisfab/redis:{{REDIS_VERSION}}-{{REDIS_ARCH}}-{{REDIS_OSNICK}} AS redis -{% if gpu_build is defined %} + +{% if gpu_build is defined and REDIS_JETSON is not defined %} FROM nvidia/cuda:{{REDIS_CUDA_VERSION}}-devel-ubuntu{{REDIS_CUDA_UBUNTUVERSION}} AS builder +{% elif REDIS_JETSON is defined %} +FROM redisfab/jetpack:4.4.1-{{REDIS_ARCH}}-l4t as builder {% else %} FROM {{REDIS_OS}} AS builder {% endif %} -ARG REDISAI_LITE -ARG PACK -ARG TEST - RUN echo "Building for {{REDIS_OSNICK}} ({{REDIS_OS}}) for {{REDIS_ARCH}} [with Redis {{REDIS_VERSION}}]" {% if gpu_build is defined %} @@ -36,22 +29,29 @@ RUN apt-get update -qq RUN apt-get upgrade -yqq RUN rm -rf /var/cache/apt -ARG DEPS_ARGS="" COPY ./get_deps.sh . -RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh {% if gpu_build is defined %}gpu {% else %} cpu {% endif %}; else env $DEPS_ARGS ./get_deps.sh {% if gpu_build is defined %} gpu {% else %} cpu {% endif %}; fi +{% if REDIS_JETSON is defined %} +{% set DEPS_ARGS = "GPU=1 JETSON=1 WITH_PT=1 WITH_TF=0 WITH_TFLITE=0 WITH_ORT=0" %} +{% set BUILD_ARGS="GPU=1 JETSON=1 WITH_TF=0 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0" %} +{% else %} +{% if gpu_build is defined %} +{% set DEPS_ARGS = "GPU=1" %} +{% endif %} +{% endif %} +RUN bash -l -c "{{DEPS_ARGS}} ./get_deps.sh" -ARG BUILD_ARGS="" ADD ./ /build -RUN bash -l -c "make -C opt build {% if gpu_build is defined %} GPU=1 {% else %} REDISAI_LITE={{REDISAI_LITE}} {% endif %}$BUILD_ARGS SHOW=1" +RUN bash -l -c "make -C opt build REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} SHOW=1" RUN mkdir -p bin/artifacts -RUN set -e ;\ - if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack {% if gpu_build is defined %} GPU=1 {% endif %} REDISAI_LITE={{REDISAI_LITE}}"; fi +{% if REDIS_PACK|int == 1 %} +RUN set -e bash -l -c "make -C opt pack {% if gpu_build is defined %} GPU=1 {% endif %} REDISAI_LITE={{REDISAI_LITE}}" +{% endif %} -RUN set -e ;\ - if [ "$TEST" = "1" ]; then \ - bash -l -c "TEST= make -C opt test {% if gpu_build is defined %}GPU=1{% endif %} REDISAI_LITE={{REDISAI_LITE}} $BUILD_ARGS NO_LFS=1" ;\ +{% if REDIS_TEST|int == 1 %} +RUN set -e;\ + bash -l -c "TEST= make -C opt test {% if gpu_build is defined %}GPU=1{% endif %} REDISAI_LITE={{REDISAI_LITE}} {{BUILD_ARGS}} NO_LFS=1" ;\ if [[ -d test/logs ]]; then \ {% if gpu_build is defined %} tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\ @@ -59,32 +59,33 @@ RUN set -e ;\ tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\ {% endif %} fi ;\ - fi + +{% endif %} #---------------------------------------------------------------------------------------------- -{% if gpu_build is defined %} +{% if gpu_build is defined and REDIS_JETSON is not defined %} FROM nvidia/cuda:{{REDIS_CUDA_VERSION}}-runtime-ubuntu{{REDIS_CUDA_UBUNTUVERSION}} {% else %} FROM redisfab/redis:{{REDIS_VERSION}}-{{REDIS_ARCH}}-{{REDIS_OSNICK}} {% endif %} -ARG PACK - RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi -ENV REDIS_MODULES /usr/lib/redis/modules -ENV LD_LIBRARY_PATH $REDIS_MODULES +{% set REDIS_MODULES = "/usr/lib/redis/modules" %} +ENV LD_LIBRARY_PATH {{REDIS_MODULES}} -RUN mkdir -p $REDIS_MODULES/ +RUN mkdir -p {{REDIS_MODULES}}/ {% if gpu_build is defined %} -COPY --from=builder /build/install-gpu/ $REDIS_MODULES/ +COPY --from=builder /build/install-gpu/ {{REDIS_MODULES}}/ {% else %} -COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ +COPY --from=builder /build/install-cpu/ {{REDIS_MODULES}}/ {% endif %} +{% if REDIS_TEST|int == 1 %} COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts +{% endif %} WORKDIR /data EXPOSE 6379 diff --git a/opt/readies b/opt/readies index 4cb9aa263..2fdf0569e 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 4cb9aa2638aab8518f63a7ecb0b97c9318035845 +Subproject commit 2fdf0569e4d662bc7c402031f2da58d1a2d3f780