Nightly python packages for python 3.14

setup.py

@@ -598,10 +598,10 @@ def finalize_options(self):
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 
 if enable_training or enable_training_apis:

tools/ci_build/build.py

@@ -1832,7 +1832,13 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
                     run_subprocess(
                         [sys.executable, "-m", "unittest", "discover", "-s", "quantization"], cwd=cwd, dll_path=dll_path
                     )
-                    if args.enable_transformers_tool_test:
+
+                    # onnx package does not support python 3.14 yet so skip the transformers tests for python 3.14.
+                    # we can remove this check when onnx package supports python 3.14.
+                    if args.enable_transformers_tool_test and (sys.version_info.major, sys.version_info.minor) < (
+                        3,
+                        14,
+                    ):
                         import google.protobuf  # noqa: PLC0415
                         import numpy  # noqa: PLC0415
 

tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml

@@ -44,7 +44,10 @@ stages:
         displayName: 'Download Pipeline Artifact - Win GPU 3.13'
         artifact: 'win_gpu_wheel_3.13'
         patterns: '*.whl'
-
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.14'
+        artifact: 'win_gpu_wheel_3.14'
+        patterns: '*.whl'
       - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \;
         displayName: 'Merge files together'
 

tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml

@@ -56,3 +56,10 @@ extends:
         EP_BUILD_FLAGS: --use_dml
         EP_NAME: directml
         cmake_build_type: ${{ parameters.cmake_build_type }}
+    - template: stages/py-win-gpu-stage.yml
+      parameters:
+        MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
+        PYTHON_VERSION: '3.14'
+        EP_BUILD_FLAGS: --use_dml
+        EP_NAME: directml
+        cmake_build_type: ${{ parameters.cmake_build_type }}

tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml

@@ -94,6 +94,8 @@ stages:
             PythonVersion: '3.12'
           Python313_x64:
             PythonVersion: '3.13'
+          Python314_x64:
+            PythonVersion: '3.14'
       variables:
         OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
         ExtraParam: ${{ parameters.build_py_parameters }}
@@ -138,7 +140,7 @@ stages:
           parameters:
             ${{if eq(variables['Build.SourceBranch'], 'refs/heads/main')}}:
               symbolExpiryTime: 60
-            includePublicSymbolServer: true            
+            includePublicSymbolServer: true
             symbolsArtifactName: onnxruntime_cpu_win_x64_$(PythonVersion)
             symbolsVersion: $(Build.BuildId)
             symbolProject: 'ONNX Runtime'
@@ -204,14 +206,21 @@ stages:
           extra_build_arg: ${{ parameters.build_py_parameters }}
           cmake_build_type: ${{ parameters.cmake_build_type }}
           python_version: '3.12'
-          
+
       - template: ../templates/py-macos.yml
         parameters:
           arch: 'arm64'
           extra_build_arg: ${{ parameters.build_py_parameters }}
           cmake_build_type: ${{ parameters.cmake_build_type }}
           python_version: '3.13'
 
+      - template: ../templates/py-macos.yml
+        parameters:
+          arch: 'arm64'
+          extra_build_arg: ${{ parameters.build_py_parameters }}
+          cmake_build_type: ${{ parameters.cmake_build_type }}
+          python_version: '3.14'
+
 - ${{ if eq(parameters.enable_linux_arm, true) }}:
   - stage: Python_Packaging_Linux_ARM
     dependsOn: []

tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml

@@ -35,6 +35,7 @@ parameters:
     - '3.11'
     - '3.12'
     - '3.13'
+    - '3.14'
 
 stages:
     - ${{ if eq(parameters.cuda_version, '12.8') }}:

tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml

@@ -50,7 +50,7 @@ stages:
     variables:
       - name: extra_build_args
         ${{ if ne(parameters.extra_build_arg, '') }}:
-          value: -x ${{ parameters.extra_build_arg }}
+          value: '-x "${{ parameters.extra_build_arg }}"'
         ${{ if eq(parameters.extra_build_arg, '') }}:
           value: ''
       - template: ../templates/common-variables.yml

tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml

@@ -175,6 +175,8 @@ stages:
 
   - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
     dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
+    # Skip this stage for Python 3.14 for now until onnx package support python 3.14.
+    condition: and(succeeded(), ne('${{ parameters.PYTHON_VERSION }}', '3.14'))
     jobs:
     - job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
       workspace:
@@ -223,6 +225,8 @@ stages:
             Copy-Item -Path $(Build.sourcesDirectory)/onnxruntime/test/python/onnx_backend_test_series.py -Destination $(Agent.TempDirectory)\ort_test_data
             Copy-Item -Recurse -Path $(Build.sourcesDirectory)/onnxruntime/test/testdata -Destination $(Agent.TempDirectory)\ort_test_data
             cd $(Agent.TempDirectory)\ort_test_data
+            python -m pip install psutil py-cpuinfo py3nvml
+            python -c "import onnxruntime; onnxruntime.print_debug_info()"
             python onnx_backend_test_series.py
           workingDirectory: '$(Build.sourcesDirectory)'
           displayName: 'Run Python Tests'

tools/ci_build/github/azure-pipelines/templates/jobs/download_win_gpu_library.yml

@@ -27,8 +27,9 @@ steps:
           set AZCOPY_AUTO_LOGIN_TYPE=AZCLI
           azcopy.exe cp --recursive https://lotusscus.blob.core.windows.net/models/cuda_sdk/v${{ parameters.CudaVersion }} $(Agent.TempDirectory)
 
+    # Since CUDA 13.0, CUDA DLLs are  in bin\x64 folder instead of bin folder for Windows.
     - powershell: |
-        Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\bin;$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\extras\CUPTI\lib64"
+        Write-Host "##vso[task.prependpath]$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\bin;$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\bin\x64;$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}\extras\CUPTI\lib64"
       displayName: 'Append CUDA SDK Directory to PATH'
 
     - task: CmdLine@2

tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml

@@ -27,7 +27,7 @@ parameters:
   displayName: QNN SDK version
   type: string
   default: 2.39.0.250926
-  
+
 - name: is1ES
   displayName: 'Whether the pipeline is running in 1ES'
   type: boolean
@@ -50,7 +50,7 @@ jobs:
   variables:
   - name: extra_build_args
     ${{ if ne(parameters.extra_build_arg, '') }}:
-      value: -x ${{ parameters.extra_build_arg }}
+      value: '-x "${{ parameters.extra_build_arg }}"'
     ${{ if eq(parameters.extra_build_arg, '') }}:
       value: ''
   steps:
@@ -79,4 +79,3 @@ jobs:
       arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args)
     env:
       ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk"
-

tools/ci_build/github/azure-pipelines/templates/py-linux.yml

@@ -61,7 +61,7 @@ jobs:
   variables:
   - name: extra_build_args
     ${{ if ne(parameters.extra_build_arg, '') }}:
-      value: '-x ${{ parameters.extra_build_arg }}'
+      value: '-x "${{ parameters.extra_build_arg }}"'
     ${{ if eq(parameters.extra_build_arg, '') }}:
       value: ''
   - name: python_exe_path
@@ -89,4 +89,4 @@ jobs:
     inputs:
       targetType: filePath
       filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
-      arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path)
+      arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path)

tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml

@@ -32,6 +32,8 @@ jobs:
         PythonVersion: '3.12'
       Python313:
         PythonVersion: '3.13'
+      Python314:
+        PythonVersion: '3.14'
   steps:
   - checkout: none
 
@@ -58,4 +60,3 @@ jobs:
         python3 -c "import onnxruntime as ort; print(ort.__version__)"
       workingDirectory: $(Pipeline.Workspace)/build/onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}
     displayName: Test Package Installation
-

tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml

@@ -35,6 +35,8 @@ jobs:
         PythonVersion: '3.12'
       Python313_x64:
         PythonVersion: '3.13'
+      Python314_x64:
+        PythonVersion: '3.14'
   variables:
     GRADLE_OPTS: '-Dorg.gradle.daemon=false'
     VSGenerator: 'Visual Studio 17 2022'
@@ -113,5 +115,3 @@ jobs:
     inputs:
       artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion)
       targetPath: '$(Build.ArtifactStagingDirectory)'
-
-

tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml

@@ -39,6 +39,8 @@ jobs:
         PythonVersion: '3.12'
       Python313_x64:
         PythonVersion: '3.13'
+      Python314_x64:
+        PythonVersion: '3.14'
   variables:
     GRADLE_OPTS: '-Dorg.gradle.daemon=false'
     VSGenerator: 'Visual Studio 17 2022'
@@ -51,9 +53,9 @@ jobs:
         parameters:
           host_cpu_arch: 'x64'
           python_version: $(PythonVersion)
-      
+
       - script: python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\windows\python\requirements.txt
-      
+
       - template: set-nightly-build-option-variable-step.yml
 
       - template: jobs/download_win_qnn_sdk.yml

tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml

@@ -21,6 +21,10 @@ steps:
         variables = {
           "PythonManylinuxDir": "/opt/python/cp313-cp313"
         }
+      elif version == "3.14":
+        variables = {
+          "PythonManylinuxDir": "/opt/python/cp314-cp314"
+        }
       else:
         raise ValueError("Unsupported Python version: '{}'".format(version))
 

tools/ci_build/github/azure-pipelines/templates/win-ci.yml

@@ -77,8 +77,8 @@ parameters:
   type: string
   default: '12.8'
   values:
-      - 13.0
       - 12.8
+      - 13.0
 
 - name: SpecificArtifact
   displayName: Use Specific Artifact

tools/ci_build/github/linux/build_linux_python_package.sh

@@ -12,9 +12,12 @@
 PYTHON_EXES=(
   "/opt/python/cp311-cp311/bin/python3.11"
   "/opt/python/cp313-cp313/bin/python3.13"
-  "/opt/python/cp313-cp313t/bin/python3.13t"
-    "/opt/python/cp312-cp312/bin/python3.12"
+  "/opt/python/cp313-cp313t/bin/python3.13"
+  "/opt/python/cp314-cp314/bin/python3.14"
+  "/opt/python/cp314-cp314t/bin/python3.14"
+  "/opt/python/cp312-cp312/bin/python3.12"
   )
+
 while getopts "d:p:x:c:e" parameter_Option
 do case "${parameter_Option}"
 in
@@ -58,7 +61,9 @@
 echo "$EXTRA_ARG"
 
 if [ "$EXTRA_ARG" != "" ]; then
-    BUILD_ARGS+=("$EXTRA_ARG")
+    # SC2206: This is intentionally unquoted to allow multiple arguments.
+    # shellcheck disable=SC2206
+    BUILD_ARGS+=($EXTRA_ARG)
 fi
 
 if [ "$ARCH" == "x86_64" ]; then
@@ -67,7 +72,8 @@
 fi
 
 if [ "$BUILD_DEVICE" == "GPU" ]; then
-    SHORT_CUDA_VERSION=$(echo $CUDA_VERSION | sed   's/\([[:digit:]]\+\.[[:digit:]]\+\)\.[[:digit:]]\+/\1/')
+    # Fix SC2086: Quote $CUDA_VERSION
+    SHORT_CUDA_VERSION=$(echo "$CUDA_VERSION" | sed   's/\([[:digit:]]\+\.[[:digit:]]\+\)\.[[:digit:]]\+/\1/')
     #Enable CUDA and TRT EPs.
     BUILD_ARGS+=("--use_cuda" "--use_tensorrt" "--cuda_version=$SHORT_CUDA_VERSION" "--tensorrt_home=/usr" "--cuda_home=/usr/local/cuda-$SHORT_CUDA_VERSION" "--cudnn_home=/usr/local/cuda-$SHORT_CUDA_VERSION" "--nvcc_threads=1" "--cmake_extra_defines" "CMAKE_CUDA_ARCHITECTURES=60-real;70-real;75-real;80-real;86-real;90a-real;90-virtual" "onnxruntime_USE_FPA_INTB_GEMM=OFF")
 fi
@@ -85,10 +91,20 @@
 
 for PYTHON_EXE in "${PYTHON_EXES[@]}"
 do
+  # Check if the Python executable or its directory exists
+  if [ ! -f "$PYTHON_EXE" ]; then
+    echo "WARNING: Python executable not found at $PYTHON_EXE. Skipping this version."
+    continue
+  fi
+
+  # Recompile the entire onnxruntime from scratch for every single Python version.
+  # TODO: It might be possible to reuse some intermediate files between different Python versions to speed up the build.
   rm -rf /build/"$BUILD_CONFIG"
+
   # that's a workaround for the issue that there's no python3 in the docker image
   # like xnnpack's cmakefile, it uses pythone3 to run a external command
   python3_dir=$(dirname "$PYTHON_EXE")
+  ls "$python3_dir"
   ${PYTHON_EXE} -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/python/requirements.txt
   PATH=$python3_dir:$PATH ${PYTHON_EXE} /onnxruntime_src/tools/ci_build/build.py "${BUILD_ARGS[@]}"
   cp /build/"$BUILD_CONFIG"/dist/*.whl /build/dist

tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 set -e -x
 pushd .
-PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13")
+PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13" "/opt/python/cp314-cp314/bin/python3.14" "/opt/python/cp314-cp314t/bin/python3.14")
 CURRENT_DIR=$(pwd)
 if ! [ -x "$(command -v protoc)" ]; then
   $CURRENT_DIR/install_protobuf.sh
 fi
 popd
 export ONNX_ML=1
@@ -13,4 +13,4 @@
 for PYTHON_EXE in "${PYTHON_EXES[@]}"
 do
   ${PYTHON_EXE} -m pip install -r requirements.txt
-done
+done

tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt

@@ -1,9 +1,10 @@
-numpy==2.2.6
+numpy==2.2.6; python_version < "3.14"
+numpy==2.3.2; python_version >= "3.14"
 mypy
 pytest
 setuptools>=68.2.2
 wheel
-onnx==1.19.1
 protobuf==4.25.8
 sympy==1.14
 flatbuffers
+onnx==1.19.1; python_version < "3.14"

tools/ci_build/github/linux/docker/scripts/install_python_deps.sh

@@ -1,19 +1,15 @@
 #!/bin/bash
 set -e -x
 
-INSTALL_DEPS_TRAINING=false
-INSTALL_DEPS_DISTRIBUTED_SETUP=false
 USE_CONDA=false
 
-while getopts p:d:v:tmuc parameter_Option
+while getopts p:d:c parameter_Option
 do case "${parameter_Option}"
 in
 p) PYTHON_VER=${OPTARG};;
 d) DEVICE_TYPE=${OPTARG};;
-v) CU_VER=${OPTARG};;
-t) INSTALL_DEPS_TRAINING=true;;
-m) INSTALL_DEPS_DISTRIBUTED_SETUP=true;;
 c) USE_CONDA=true;;
+*) echo "Invalid option: -${OPTARG}" >&2; exit 1 ;;
 esac
 done
 
@@ -37,4 +33,5 @@ fi
 
 export ONNX_ML=1
 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
-${PYTHON_EXE} -m pip install -r ${0/%install_python_deps\.sh/requirements\.txt}
+
+${PYTHON_EXE} -m pip install -r "${0/%install_python_deps\.sh/requirements\.txt}"