append faster-coco-eval as base backend for COCODetection

.circleci/config.yml

@@ -29,7 +29,7 @@ jobs:
         type: string
       paddle:
         type: string
-        default: "2.2.2"
+        default: "2.3.2"
       tensorflow:
         type: string
         default: "2.4"
@@ -108,47 +108,53 @@ jobs:
           command: |
             mpirun -np 2 pytest -vv --capture=no --with-mpi tests/test_core/test_dist_backends
 
-  build_cu112:
+  build_cuda:
+    parameters:
+      torch:
+        type: string
+      cuda:
+        type: enum
+        enum: ["11.7", "11.8"]
+      cudnn:
+        type: integer
+        default: 8
     machine:
-      image: ubuntu-2004-cuda-11.2:202103-01
+      image: linux-cuda-11:default
+      # docker_layer_caching: true
     resource_class: gpu.nvidia.small
     steps:
       - checkout
       - run:
-          # CUDNN is required for Paddle to use GPU, otherwise, the test cases will report an error.
-          name: Install CUDNN
+          # CLoning repos in VM since Docker doesn't have access to the private key
+          name: Clone Repos
+          command: |
+            git clone -b main --depth 1 ssh://[email protected]/open-mmlab/mmengine.git /home/circleci/mmengine
+      - run:
+          name: Install nvidia-container-toolkit and Restart Docker
           command: |
-            OS=ubuntu2004
-            cudnn_version=8.1.0.77
-            cuda_version=cuda11.2
-            wget https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
-            sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
-            sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/3bf863cc.pub
-            sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/ /"
             sudo apt-get update
-            sudo apt-get install libcudnn8=${cudnn_version}-1+${cuda_version}
-            sudo apt-get install libcudnn8-dev=${cudnn_version}-1+${cuda_version}
+            sudo apt-get install -y nvidia-container-toolkit
+            sudo systemctl restart docker
       - run:
-          name: Install PyTorch, Paddle and OneFlow via pip
+          name: Build Docker image
           command: |
-            pyenv global 3.9.2
-            pip install --upgrade pip
-            python -V
-            pip install torch==1.7.1+cu110 -f https://download.pytorch.org/whl/torch_stable.html
-            pip install paddlepaddle-gpu==2.3.2.post112 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html
-            # Install Oneflow often failed, we just skip it if failed.
-            pip install --pre oneflow -f https://staging.oneflow.info/branch/master/cu112 "numpy<1.24.0" || true
+            docker build .circleci/docker -t mmeval:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >>
+            docker run --gpus all -t -d -v /home/circleci/project:/mmeval -v /home/circleci/mmengine:/mmengine -w /mmeval --name mmeval mmeval:gpu
+            docker exec mmeval apt-get install -y git
       - run:
-          name: Install mmeval and dependencies
+          name: Install mmeval dependencies
           command: |
-            pip install -r requirements/tests.txt
-            pip install -r requirements/optional.txt
-            pip install -e .
+            docker exec mmeval pip install paddlepaddle-gpu==2.3.2.post112 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html
+            docker exec mmeval pip install -r requirements/tests.txt
+            docker exec mmeval pip install -r requirements/optional.txt
+      - run:
+          name: Build and install
+          command: |
+            docker exec mmeval pip install -e .
       - run:
           name: Run unittests
           command: |
-            pytest -vv tests/
-
+            docker exec mmeval python -m pytest -vv tests/
 
 workflows:
   unit_tests:
@@ -198,6 +204,9 @@ workflows:
             - build_cpu_torch1.8_tf2.7_py3.9
             - build_cpu_torch1.9_paddle2.3_tf2.8_py3.8
             - build_cpu_torch1.9_paddle2.10_py3.9
-      - build_cu112:
+      - build_cuda:
+          name: build_cuda117_torch1.8_paddle2.3
+          torch: 1.8.1
+          cuda: "11.7"
           requires:
-            - build_mpirun_and_tf_2.7
+            - lint

.circleci/docker/Dockerfile

@@ -0,0 +1,11 @@
+ARG PYTORCH="1.8.1"
+ARG CUDA="10.2"
+ARG CUDNN="7"
+
+FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel
+
+# To fix GPG key error when running apt-get update
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
+
+RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx

.pre-commit-config-zh-cn.yaml

@@ -54,9 +54,10 @@ repos:
       - id: check-copyright
         args: ["mmeval"]
   - repo: https://gitee.com/openmmlab/mirrors-mypy
-    rev: v0.812
+    rev: v0.981
     hooks:
       - id: mypy
+        additional_dependencies: ["types-setuptools", "types-PyYAML"] # error: Library stubs not installed for "yaml" (or incompatible with Python 3.10)
         exclude: |-
           (?x)(
               ^test

.pre-commit-config.yaml

@@ -54,9 +54,10 @@ repos:
       - id: check-copyright
         args: ["mmeval"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v0.981
     hooks:
       - id: mypy
+        additional_dependencies: ["types-setuptools", "types-PyYAML"] # error: Library stubs not installed for "yaml" (or incompatible with Python 3.10)
         exclude: |-
           (?x)(
               ^test

mmeval/fileio/io.py

@@ -431,8 +431,12 @@ def load(file, file_format=None, backend_args=None, **kwargs):
     if isinstance(file, str):
         file_backend = get_file_backend(file, backend_args=backend_args)
         if handler.str_like:
-            with StringIO(file_backend.get_text(file)) as f:
-                obj = handler.load_from_fileobj(f, **kwargs)
+            try:
+                with StringIO(file_backend.get_text(file)) as f:
+                    obj = handler.load_from_fileobj(f, **kwargs)
+            except FileNotFoundError as e:
+                raise FileNotFoundError('{} {} {} {}'.format(
+                    file, handler, file_format, e))
         else:
             with BytesIO(file_backend.get(file)) as f:
                 obj = handler.load_from_fileobj(f, **kwargs)

mmeval/metrics/coco_detection.py

@@ -23,6 +23,13 @@
 except ImportError:
     HAS_COCOAPI = False
 
+try:
+    from mmeval.metrics.utils.faster_coco_wrapper import (FasterCOCO,
+                                                          FasterCOCOeval)
+    HAS_FasterCOCOAPI = True
+except ImportError:
+    HAS_FasterCOCOAPI = False
+
 
 class COCODetection(BaseMetric):
     """COCO object detection task evaluation metric.
@@ -60,6 +67,7 @@ class COCODetection(BaseMetric):
         backend_args (dict, optional): Arguments to instantiate the
             preifx of uri corresponding backend. Defaults to None.
         print_results (bool): Whether to print the results. Defaults to True.
+        faster (bool): Use faster-coco-eval for faster calculations. Defaults to True.
         logger (Logger, optional): logger used to record messages. When set to
             ``None``, the default logger will be used.
             Defaults to None.
@@ -162,12 +170,22 @@ def __init__(self,
                  gt_mask_area: bool = True,
                  backend_args: Optional[dict] = None,
                  print_results: bool = True,
+                 faster: bool = True,
                  **kwargs) -> None:
         if not HAS_COCOAPI:
             raise RuntimeError('Failed to import `COCO` and `COCOeval` from '
                                '`mmeval.utils.coco_wrapper`. '
                                'Please try to install official pycocotools by '
                                '"pip install pycocotools"')
+
+        if faster and not HAS_FasterCOCOAPI:
+            warnings.warn(
+                'Failed to import `FasterCOCO` and `FasterCOCOeval` from '
+                '`mmeval.utils.faster_coco_wrapper`. '
+                'Please try to install official faster-coco-eval by '
+                '"pip install faster-coco-eval"')
+            faster = False
+
         super().__init__(**kwargs)
         # coco evaluation metrics
         self.metrics = metric if isinstance(metric, list) else [metric]
@@ -199,6 +217,7 @@ def __init__(self,
         self.iou_thrs = iou_thrs
         self.metric_items = metric_items
         self.print_results = print_results
+        self.faster = faster
         self.format_only = format_only
         if self.format_only:
             assert outfile_prefix is not None, 'outfile_prefix must be not'
@@ -209,12 +228,15 @@ def __init__(self,
 
         # if ann_file is not specified,
         # initialize coco api with the converted dataset
-        self._coco_api: Optional[COCO]  # type: ignore
+        self._coco_api: Optional[Union[COCO, FasterCOCO]]  # type: ignore
         if ann_file is not None:
             with get_local_path(
                     filepath=ann_file,
                     backend_args=backend_args) as local_path:
-                self._coco_api = COCO(annotation_file=local_path)
+                if self.faster:
+                    self._coco_api = FasterCOCO(annotation_file=local_path)
+                else:
+                    self._coco_api = COCO(annotation_file=local_path)
         else:
             self._coco_api = None
 
@@ -510,7 +532,11 @@ def compute_metric(self, results: list) -> dict:
             self.logger.info('Converting ground truth to coco format...')
             coco_json_path = self.gt_to_coco_json(
                 gt_dicts=gts, outfile_prefix=outfile_prefix)
-            self._coco_api = COCO(coco_json_path)
+
+            if self.faster:
+                self._coco_api = FasterCOCO(coco_json_path)
+            else:
+                self._coco_api = COCO(coco_json_path)
 
         # handle lazy init
         if len(self.cat_ids) == 0:
@@ -553,8 +579,10 @@ def compute_metric(self, results: list) -> dict:
                 self.logger.warning('The testing results of the '
                                     'whole dataset is empty.')
                 break
-
-            coco_eval = COCOeval(self._coco_api, coco_dt, iou_type)
+            if self.faster:
+                coco_eval = FasterCOCOeval(self._coco_api, coco_dt, iou_type)
+            else:
+                coco_eval = COCOeval(self._coco_api, coco_dt, iou_type)
 
             coco_eval.params.catIds = self.cat_ids
             coco_eval.params.imgIds = self.img_ids

mmeval/metrics/utils/faster_coco_wrapper.py

@@ -0,0 +1,22 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from faster_coco_eval import COCO as _COCO
+from faster_coco_eval import COCOeval_faster as _faster_COCOeval
+
+from mmeval.metrics.utils.coco_wrapper import COCO
+
+
+class FasterCOCO(COCO, _COCO):
+    """This class is almost the same as official pycocotools package and
+    faster-coco-eval package.
+
+    It implements some snake case function aliases. So that the COCO class has
+    the same interface as LVIS class.
+
+    Args:
+        annotation_file (str, optional): Path of annotation file.
+            Defaults to None.
+    """
+
+
+# just for the ease of import
+FasterCOCOeval = _faster_COCOeval

requirements/optional.txt

@@ -1,2 +1,3 @@
+faster-coco-eval @ git+https://github.com/MiXaiLL76/faster_coco_eval@dev
 pycocotools
 shapely

tests/test_fileio/test_io.py

@@ -315,7 +315,7 @@ def test_list_dir_or_file():
 def test_load():
     # invalid file format
     with pytest.raises(TypeError, match='Unsupported format'):
-        fileio.load('filename.txt')
+        fileio.load('filename.ttx')
 
     # file is not a valid input
     with pytest.raises(