Skip to content

Commit 0987b6a

Browse files
maysunfaisalmaysunfaisal
authored
Merge pull request #17 from maysunfaisal/vllm-0.6.4-1
Upload vllm 0.6.4 model server
2 parents 3993403 + 67c9be1 commit 0987b6a

29 files changed

+3014
-0
lines changed

‎model-servers/vllm/0.6.4/Containerfile‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
FROM registry.access.redhat.com/ubi9/python-311 as cuda-runtime
2+
3+
###################################################################################################
4+
# CUDA 12.1 Layer, from https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/12.1.1 #
5+
###################################################################################################
6+
7+
# Base
8+
USER 0
9+
10+
ENV NVARCH x86_64
11+
ENV NVIDIA_REQUIRE_CUDA "cuda>=12.1 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=525,driver<526 brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526 brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526 brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526"
12+
ENV NV_CUDA_CUDART_VERSION 12.1.105-1
13+
14+
COPY cuda.repo-x86_64 /etc/yum.repos.d/cuda.repo
15+
16+
RUN NVIDIA_GPGKEY_SUM=d0664fbbdb8c32356d45de36c5984617217b2d0bef41b93ccecd326ba3b80c87 && \
17+
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel9/${NVARCH}/D42D0685.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
18+
echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c --strict -
19+
20+
ENV CUDA_VERSION 12.1.1
21+
22+
# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
23+
RUN yum upgrade -y && yum install -y \
24+
cuda-cudart-12-1-${NV_CUDA_CUDART_VERSION} \
25+
cuda-compat-12-1 \
26+
&& ln -s cuda-12.1 /usr/local/cuda \
27+
&& yum -y clean all --enablerepo='*' && \
28+
rm -rf /var/cache/dnf && \
29+
find /var/log -type f -name "*.log" -exec rm -f {} \;
30+
31+
# nvidia-docker 1.0
32+
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
33+
echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
34+
35+
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
36+
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
37+
38+
COPY NGC-DL-CONTAINER-LICENSE /
39+
40+
# nvidia-container-runtime
41+
ENV NVIDIA_VISIBLE_DEVICES all
42+
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
43+
44+
# Runtime
45+
ENV NV_CUDA_LIB_VERSION 12.1.1-1
46+
47+
ENV NV_NVTX_VERSION 12.1.105-1
48+
ENV NV_LIBNPP_VERSION 12.1.0.40-1
49+
ENV NV_LIBNPP_PACKAGE libnpp-12-1-${NV_LIBNPP_VERSION}
50+
ENV NV_LIBCUBLAS_VERSION 12.1.3.1-1
51+
ENV NV_LIBNCCL_PACKAGE_NAME libnccl
52+
ENV NV_LIBNCCL_PACKAGE_VERSION 2.17.1-1
53+
ENV NV_LIBNCCL_VERSION 2.17.1
54+
ENV NCCL_VERSION 2.17.1
55+
ENV NV_LIBNCCL_PACKAGE ${NV_LIBNCCL_PACKAGE_NAME}-${NV_LIBNCCL_PACKAGE_VERSION}+cuda12.1
56+
57+
RUN yum install -y \
58+
cuda-libraries-12-1-${NV_CUDA_LIB_VERSION} \
59+
cuda-nvtx-12-1-${NV_NVTX_VERSION} \
60+
${NV_LIBNPP_PACKAGE} \
61+
libcublas-12-1-${NV_LIBCUBLAS_VERSION} \
62+
${NV_LIBNCCL_PACKAGE} \
63+
&& yum clean all \
64+
&& rm -rf /var/cache/yum/*
65+
66+
# Set this flag so that libraries can find the location of CUDA
67+
ENV XLA_FLAGS=--xla_gpu_cuda_data_dir=/usr/local/cuda
68+
69+
# CUDA Devel image
70+
FROM cuda-runtime as cuda-devel
71+
ENV NVIDIA_VISIBLE_DEVICES all
72+
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
73+
74+
ENV NV_CUDA_LIB_VERSION 12.1.1-1
75+
ENV NV_NVPROF_VERSION 12.1.105-1
76+
ENV NV_NVPROF_DEV_PACKAGE cuda-nvprof-12-1-${NV_NVPROF_VERSION}
77+
ENV NV_CUDA_CUDART_DEV_VERSION 12.1.105-1
78+
ENV NV_NVML_DEV_VERSION 12.1.105-1
79+
ENV NV_LIBCUBLAS_DEV_VERSION 12.1.3.1-1
80+
ENV NV_LIBNPP_DEV_VERSION 12.1.0.40-1
81+
ENV NV_LIBNPP_DEV_PACKAGE libnpp-devel-12-1-${NV_LIBNPP_DEV_VERSION}
82+
ENV NV_LIBNCCL_DEV_PACKAGE_NAME libnccl-devel
83+
ENV NV_LIBNCCL_DEV_PACKAGE_VERSION 2.17.1-1
84+
ENV NCCL_VERSION 2.17.1
85+
ENV NV_LIBNCCL_DEV_PACKAGE ${NV_LIBNCCL_DEV_PACKAGE_NAME}-${NV_LIBNCCL_DEV_PACKAGE_VERSION}+cuda12.1
86+
ENV NV_CUDA_NSIGHT_COMPUTE_VERSION 12.1.1-1
87+
ENV NV_CUDA_NSIGHT_COMPUTE_DEV_PACKAGE cuda-nsight-compute-12-1-${NV_CUDA_NSIGHT_COMPUTE_VERSION}
88+
89+
90+
RUN yum install -y \
91+
make \
92+
findutils \
93+
cuda-command-line-tools-12-1-${NV_CUDA_LIB_VERSION} \
94+
cuda-libraries-devel-12-1-${NV_CUDA_LIB_VERSION} \
95+
cuda-minimal-build-12-1-${NV_CUDA_LIB_VERSION} \
96+
cuda-cudart-devel-12-1-${NV_CUDA_CUDART_DEV_VERSION} \
97+
${NV_NVPROF_DEV_PACKAGE} \
98+
cuda-nvml-devel-12-1-${NV_NVML_DEV_VERSION} \
99+
libcublas-devel-12-1-${NV_LIBCUBLAS_DEV_VERSION} \
100+
${NV_LIBNPP_DEV_PACKAGE} \
101+
${NV_LIBNCCL_DEV_PACKAGE} \
102+
${NV_CUDA_NSIGHT_COMPUTE_DEV_PACKAGE} \
103+
&& yum clean all \
104+
&& rm -rf /var/cache/yum/*
105+
106+
ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
107+
108+
#############################################
109+
# End of CUDA 12.1 Layer #
110+
#############################################
111+
112+
###################################
113+
# vLLM install in build container #
114+
###################################
115+
116+
FROM cuda-devel as vllm-install
117+
118+
WORKDIR /opt/app-root/src
119+
120+
USER 1001
121+
122+
COPY --chown=1001:0 requirements.txt ./
123+
124+
RUN pip install --no-cache-dir -r requirements.txt && \
125+
rm -f requirements.txt && \
126+
# Install flash-attn from PyPI \
127+
pip install flash-attn==2.5.8 --no-build-isolation && \
128+
# Correction for FIPS mode \
129+
sed -i s/md5/sha1/g /opt/app-root/lib64/python3.11/site-packages/triton/runtime/jit.py && \
130+
# Fix permissions to support pip in Openshift environments \
131+
chmod -R g+w /opt/app-root/lib/python3.11/site-packages && \
132+
fix-permissions /opt/app-root -P
133+
134+
##################
135+
# vLLM container #
136+
##################
137+
138+
FROM cuda-runtime as vllm-container
139+
140+
WORKDIR /opt/app-root/src
141+
142+
COPY --from=vllm-install --chown=1001:0 /opt/app-root/lib64/python3.11/site-packages /opt/app-root/lib64/python3.11/site-packages
143+
COPY --from=vllm-install --chown=1001:0 /opt/app-root/src/.config/vllm/nccl/cu12/libnccl.so.2* /usr/local/lib/libnccl.so.2
144+
145+
# Fix VLLM_NCCL_SO_PATH
146+
ENV VLLM_NCCL_SO_PATH=/usr/local/lib/libnccl.so.2
147+
148+
USER 1001
149+
150+
EXPOSE 8000
151+
152+
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
153+

0 commit comments

Comments
 (0)