test unidist

Signed-off-by: Igoshev, Iaroslav <[email protected]>
YarShev · Oct 2, 2023 · a187357 · a187357
1 parent 02b1323
commit a187357
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 16 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -97,7 +97,7 @@ jobs:
         run: |
           MODIN_ENGINE=dask python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
           MODIN_ENGINE=ray python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
-          MODIN_ENGINE=unidist UNIDIST_BACKEND=mpi mpiexec -n 1 python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
+          MODIN_ENGINE=unidist UNIDIST_BACKEND=mpi mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
 
   test-internals:
     needs: [lint-flake8, lint-black-isort]
@@ -360,26 +360,26 @@ jobs:
         run: |
           sudo docker pull postgres
           sudo docker run --name some-postgres -e POSTGRES_USER=sa -e POSTGRES_PASSWORD=Strong.Pwd-123 -e POSTGRES_DB=postgres -d -p 2345:5432 postgres
-      - run: mpiexec -n 1 python -m pytest modin/pandas/test/internals/test_benchmark_mode.py
-      - run: mpiexec -n 1 python -m pytest modin/pandas/test/internals/test_repartition.py
-      - run: mpiexec -n 1 python -m pytest modin/test/test_partition_api.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/pandas/test/internals/test_benchmark_mode.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/pandas/test/internals/test_repartition.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/test/test_partition_api.py
       - uses: ./.github/actions/run-core-tests
         with:
-          runner: mpiexec -n 1 python -m pytest
+          runner: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest
           parallel: ""
-      - run: mpiexec -n 1 python -m pytest modin/numpy/test
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/numpy/test
       - run: chmod +x ./.github/workflows/sql_server/set_up_sql_server.sh
       - run: ./.github/workflows/sql_server/set_up_sql_server.sh
       # need an extra argument "genv" to set environment variables for mpiexec. We need
       # these variables to test writing to the mock s3 filesystem.
-      - run: mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest modin/pandas/test/test_io.py --verbose
-      - run: mpiexec -n 1 python -m pytest modin/experimental/pandas/test/test_io_exp.py
-      - run: mpiexec -n 1 python -m pytest modin/experimental/sql/test/test_sql.py
-      - run: mpiexec -n 1 python -m pytest modin/test/interchange/dataframe_protocol/test_general.py
-      - run: mpiexec -n 1 python -m pytest modin/test/interchange/dataframe_protocol/pandas/test_protocol.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 -x AWS_ACCESS_KEY_ID=foobar_key -x AWS_SECRET_ACCESS_KEY=foobar_secret python -m pytest modin/pandas/test/test_io.py --verbose
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/experimental/pandas/test/test_io_exp.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/experimental/sql/test/test_sql.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/test/interchange/dataframe_protocol/test_general.py
+      - run: mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/test/interchange/dataframe_protocol/pandas/test_protocol.py
       - run: |
           python -m pip install lazy_import
-          mpiexec -n 1 python -m pytest modin/pandas/test/integrations/
+          mpiexec --oversubscribe -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 python -m pytest modin/pandas/test/integrations/
       - uses: ./.github/actions/upload-coverage
 
   test-all:
@@ -533,7 +533,7 @@ jobs:
             shell-ex: "python -m pytest"
             if: needs.execution-filter.dask != 'true'
           - name: unidist
-            shell-ex: "mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest"
+            shell-ex: "mpiexec --oversubscribe -n 1 -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -x AWS_ACCESS_KEY_ID=foobar_key -x AWS_SECRET_ACCESS_KEY=foobar_secret python -m pytest"
             if: needs.execution-filter.unidist != 'true'
     runs-on: ${{ matrix.os }}-latest
     defaults:

diff --git a/requirements/env_unidist_linux.yml b/requirements/env_unidist_linux.yml
@@ -7,8 +7,10 @@ dependencies:
   # required dependencies
   - pandas>=2.1,<2.2
   - numpy>=1.22.4
-  - unidist-mpi>=0.2.1
-  - mpich
+  - cython
+  - mpi4py
+  - openmpi
+  - msgpack-python
   - fsspec>=2022.05.0
   - packaging>=21.0
   - psutil>=5.8.0
@@ -59,3 +61,4 @@ dependencies:
       - connectorx>=0.2.6a4
       # The `numpydoc` version should match the version installed in the `lint-pydocstyle` job of the CI.
       - numpydoc==1.1.0
+      - unidist@git+https://github.com/YarShev/unidist.git
diff --git a/requirements/env_unidist_win.yml b/requirements/env_unidist_win.yml
@@ -7,8 +7,10 @@ dependencies:
   # required dependencies
   - pandas>=2.1,<2.2
   - numpy>=1.22.4
-  - unidist-mpi>=0.2.1
+  - cython
+  - mpi4py
   - msmpi
+  - msgpack-python
   - fsspec>=2022.05.0
   - packaging>=21.0
   - psutil>=5.8.0
@@ -59,3 +61,4 @@ dependencies:
       - connectorx>=0.2.6a4
       # The `numpydoc` version should match the version installed in the `lint-pydocstyle` job of the CI.
       - numpydoc==1.1.0
+      - unidist@git+https://github.com/YarShev/unidist.git