Update dependencies for Annif compatibility

.github/workflows/pytest.yml

@@ -6,7 +6,7 @@ jobs:
   Ubuntu-Python-Unit-Test:
     name: Ubuntu Python Unit Tests
     strategy:
-      max-parallel: 4
+      max-parallel: 8
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12", "3.13", "3.14"]
@@ -44,13 +44,20 @@ jobs:
     container: amazonlinux:2023
 
     steps:
+    - name: Install required tools for checkout
+      run: |
+        yum -y update
+        yum -y install \
+          tar \
+          gzip
     - uses: actions/checkout@v6
     - name: Install dependencies
       run: |
         yum -y update
         yum -y groupinstall 'Development Tools'
-        yum install spal-release -y
-        yum install openblas-devel -y
+        yum -y install \
+          spal-release \
+          openblas-devel
     - name: Set up Python ${{ matrix.python-version }} via miniconda
       run: |
         curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh;

pecos/utils/smat_util.py

@@ -14,7 +14,7 @@
 import scipy.sparse as smat
 
 
-def cs_matrix(arg1, mat_type, shape=None, dtype=None, copy=False, check_contents=False):
+def cs_matrix(arg1, mat_type, shape=None, copy=None, dtype=None, check_contents=False):
     """Custom compressed sparse matrix constructor that allows indices and indptr to be stored in different types.
 
     Args:
@@ -29,8 +29,8 @@ def cs_matrix(arg1, mat_type, shape=None, dtype=None, copy=False, check_contents
         compressed sparse matrix in mat_type
     """
     (data, indices, indptr) = arg1
-    indices_dtype = smat.sputils.get_index_dtype(indices, check_contents=check_contents)
-    indptr_dtype = smat.sputils.get_index_dtype(indptr, check_contents=check_contents)
+    indices_dtype = smat.get_index_dtype(indices, check_contents=check_contents)
+    indptr_dtype = smat.get_index_dtype(indptr, check_contents=check_contents)
 
     ret = mat_type(shape, dtype=dtype)
     # Read matrix dimensions given, if any
@@ -51,7 +51,7 @@ def cs_matrix(arg1, mat_type, shape=None, dtype=None, copy=False, check_contents
     return ret
 
 
-def csr_matrix(arg1, shape=None, dtype=None, copy=False):
+def csr_matrix(arg1, shape=None, dtype=None, copy=None):
     """Custom csr_matrix constructor that allows indices and indptr to be stored in different types.
 
     Args:
@@ -66,7 +66,7 @@ def csr_matrix(arg1, shape=None, dtype=None, copy=False):
     return cs_matrix(arg1, smat.csr_matrix, shape=shape, dtype=dtype, copy=copy)
 
 
-def csc_matrix(arg1, shape=None, dtype=None, copy=False):
+def csc_matrix(arg1, shape=None, dtype=None, copy=None):
     """Custom csc_matrix constructor that allows indices and indptr to be stored in different types.
 
     Args:
@@ -368,7 +368,7 @@ def vstack_csr(matrices, dtype=None):
     # infer result dtypes from inputs
     int32max = np.iinfo(np.int32).max
     if dtype is None:
-        dtype = smat.sputils.upcast(*[mat.dtype for mat in matrices])
+        dtype = np.result_type(*[mat.dtype for mat in matrices])
     indices_dtype = np.int64 if nr_cols > int32max else np.int32
     indptr_dtype = np.int64 if total_nnz > int32max else np.int32
 
@@ -417,7 +417,7 @@ def hstack_csr(matrices, dtype=None):
     # infer result dtypes from inputs
     int32max = np.iinfo(np.int32).max
     if dtype is None:
-        dtype = smat.sputils.upcast(*[mat.dtype for mat in matrices])
+        dtype = np.result_type(*[mat.dtype for mat in matrices])
     indices_dtype = np.int64 if nr_rows > int32max else np.int32
     indptr_dtype = np.int64 if total_nnz > int32max else np.int32
 
@@ -465,7 +465,7 @@ def block_diag_csr(matrices, dtype=None):
     # infer result dtypes from inputs
     int32max = np.iinfo(np.int32).max
     if dtype is None:
-        dtype = smat.sputils.upcast(*[mat.dtype for mat in matrices])
+        dtype = np.result_type(*[mat.dtype for mat in matrices])
     indices_dtype = np.int64 if total_rows > int32max else np.int32
     indptr_dtype = np.int64 if total_nnz > int32max else np.int32
 

pecos/xmc/xtransformer/matcher.py

@@ -26,7 +26,8 @@
 from pecos.xmc import MLModel, MLProblem, PostProcessor
 from sklearn.preprocessing import normalize as sk_normalize
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
-from transformers import AdamW, AutoConfig, get_scheduler, BatchEncoding
+from torch.optim import AdamW
+from transformers import AutoConfig, get_scheduler, BatchEncoding
 
 from .module import XMCLabelTensorizer, XMCTextTensorizer, XMCTextDataset
 from .network import ENCODER_CLASSES, HingeLoss, TransformerLinearXMCHead
@@ -520,8 +521,8 @@ def text_to_tensor(self, corpus, max_length=None):
         os.environ["TOKENIZERS_PARALLELISM"] = "true"
         LOGGER.info("***** Encoding data len={} truncation={}*****".format(len(corpus), max_length))
         t_start = time.time()
-        feature_tensors = self.text_tokenizer.batch_encode_plus(
-            batch_text_or_text_pairs=corpus,
+        feature_tensors = self.text_tokenizer(
+            corpus,
             **self._get_tokenizer_config(max_length=max_length),
         )
         os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -811,7 +812,7 @@ def _predict(
                         cpred_csr = smat_util.sorted_csr(cpred_csr, only_topk=local_topk)
                         batch_cpred.append(cpred_csr)
                     else:
-                        cur_act_labels = csr_codes_next[inputs["instance_number"].cpu()]
+                        cur_act_labels = csr_codes_next[inputs["instance_number"].cpu().tolist()]
                         nnz_of_insts = cur_act_labels.indptr[1:] - cur_act_labels.indptr[:-1]
                         inst_idx = np.repeat(
                             np.arange(cur_batch_size, dtype=np.uint32), nnz_of_insts

pecos/xmc/xtransformer/network.py

@@ -32,24 +32,10 @@
     DistilBertTokenizerFast,
     DistilBertPreTrainedModel,
 )
-from transformers.file_utils import add_start_docstrings
-from transformers.modeling_utils import SequenceSummary
-
-from transformers.models.bert.modeling_bert import BERT_INPUTS_DOCSTRING, BERT_START_DOCSTRING
-from transformers.models.roberta.modeling_roberta import (
-    RobertaPreTrainedModel,
-    ROBERTA_INPUTS_DOCSTRING,
-    ROBERTA_START_DOCSTRING,
-)
-from transformers.models.xlm_roberta.modeling_xlm_roberta import XLM_ROBERTA_START_DOCSTRING
-from transformers.models.xlnet.modeling_xlnet import (
-    XLNET_INPUTS_DOCSTRING,
-    XLNET_START_DOCSTRING,
-)
-from transformers.models.distilbert.modeling_distilbert import (
-    DISTILBERT_INPUTS_DOCSTRING,
-    DISTILBERT_START_DOCSTRING,
-)
+
+from transformers.utils import auto_docstring
+from transformers.models.xlnet.modeling_xlnet import XLNetSequenceSummary as SequenceSummary
+from transformers.models.roberta.modeling_roberta import RobertaPreTrainedModel
 
 
 class TransformerModelClass(object):
@@ -218,10 +204,7 @@ def forward(self, pooled_output=None, output_indices=None, num_device=1):
         return W_act, b_act
 
 
-@add_start_docstrings(
-    """Bert Model with mutli-label classification head on top for XMC.\n""",
-    BERT_START_DOCSTRING,
-)
+@auto_docstring(custom_intro="""Bert Model with mutli-label classification head on top for XMC.\n""")
 class BertForXMC(BertPreTrainedModel):
     """
     Examples:
@@ -239,12 +222,13 @@ def __init__(self, config):
         self.bert = BertModel(config)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
 
+        self.post_init()
         self.init_weights()
 
     def init_from(self, model):
         self.bert = model.bert
 
-    @add_start_docstrings(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
+    @auto_docstring
     def forward(
         self,
         input_ids=None,
@@ -256,6 +240,12 @@ def forward(
         label_embedding=None,
     ):
         r"""
+        Args:
+            label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*):
+                Pre-computed label embeddings for the multi-label classification head.
+            head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*):
+                Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1.
+
         Returns:
           :obj:`dict` containing:
                 {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label,
@@ -289,10 +279,7 @@ def forward(
         }
 
 
-@add_start_docstrings(
-    """Roberta Model with mutli-label classification head on top for XMC.\n""",
-    ROBERTA_START_DOCSTRING,
-)
+@auto_docstring(custom_intro="""Roberta Model with mutli-label classification head on top for XMC.\n""")
 class RobertaForXMC(RobertaPreTrainedModel):
     """
     Examples:
@@ -310,12 +297,13 @@ def __init__(self, config):
         self.roberta = RobertaModel(config)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
 
+        self.post_init()
         self.init_weights()
 
     def init_from(self, model):
         self.roberta = model.roberta
 
-    @add_start_docstrings(ROBERTA_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
+    @auto_docstring
     def forward(
         self,
         input_ids=None,
@@ -327,6 +315,12 @@ def forward(
         label_embedding=None,
     ):
         r"""
+        Args:
+            label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*):
+                Pre-computed label embeddings for the multi-label classification head.
+            head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*):
+                Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1.
+
         Returns:
           :obj:`dict` containing:
                 {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label,
@@ -361,10 +355,7 @@ def forward(
         }
 
 
-@add_start_docstrings(
-    """XLM-Roberta Model with mutli-label classification head on top for XMC.\n""",
-    XLM_ROBERTA_START_DOCSTRING,
-)
+@auto_docstring(custom_intro="""XLM-Roberta Model with mutli-label classification head on top for XMC.\n""")
 class XLMRobertaForXMC(RobertaForXMC):
     """
     This class overrides :class:`RobertaForXMC`. Please check the superclass for the appropriate
@@ -374,10 +365,7 @@ class XLMRobertaForXMC(RobertaForXMC):
     config_class = XLMRobertaConfig  # type: ignore
 
 
-@add_start_docstrings(
-    """XLNet Model with mutli-label classification head on top for XMC.\n""",
-    XLNET_START_DOCSTRING,
-)
+@auto_docstring(custom_intro="""XLNet Model with mutli-label classification head on top for XMC.\n""")
 class XLNetForXMC(XLNetPreTrainedModel):
     """
     Examples:
@@ -395,12 +383,13 @@ def __init__(self, config):
         self.transformer = XLNetModel(config)
         self.sequence_summary = SequenceSummary(config)
 
+        self.post_init()
         self.init_weights()
 
     def init_from(self, model):
         self.transformer = model.transformer
 
-    @add_start_docstrings(XLNET_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
+    @auto_docstring
     def forward(
         self,
         input_ids=None,
@@ -415,6 +404,21 @@ def forward(
         label_embedding=None,
     ):
         r"""
+        Args:
+            mems (`List[torch.FloatTensor]`, *optional*):
+                Pre-computed hidden-states (key and value tensors) of the self-attention blocks.
+                Can be used to speed up sequential decoding. The `mems` are returned when `use_mems=True` is passed.
+            perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
+                Permutation mask for the attention heads. Can be used for language modeling.
+            target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
+                Mask to map target tokens to input indices. Required for target prediction in PLM tasks.
+            input_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices (alternative to attention_mask).
+            label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*):
+                Pre-computed label embeddings for the multi-label classification head.
+            head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*):
+                Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1.
+
         Returns:
           :obj:`dict` containing:
                 {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label,
@@ -449,10 +453,7 @@ def forward(
         }
 
 
-@add_start_docstrings(
-    """DistilBert Model with mutli-label classification head on top for XMC.\n""",
-    DISTILBERT_START_DOCSTRING,
-)
+@auto_docstring(custom_intro="""DistilBert Model with mutli-label classification head on top for XMC.\n""")
 class DistilBertForXMC(DistilBertPreTrainedModel):
     """
     Examples:
@@ -470,12 +471,13 @@ def __init__(self, config):
         self.distilbert = DistilBertModel(config)
         self.dropout = nn.Dropout(config.dropout)
 
+        self.post_init()
         self.init_weights()
 
     def init_from(self, model):
         self.distilbert = model.distilbert
 
-    @add_start_docstrings(DISTILBERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
+    @auto_docstring
     def forward(
         self,
         input_ids=None,
@@ -486,6 +488,12 @@ def forward(
         label_embedding=None,
     ):
         r"""
+        Args:
+            label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*):
+                Pre-computed label embeddings for the multi-label classification head.
+            head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*):
+                Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1.
+
         Returns:
           :obj:`dict` containing:
                 {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label,

setup.py

@@ -84,21 +84,22 @@ def get_version(cls):
 with open("README.md", "r", encoding="utf-8") as f:
     long_description = f.read()
 
-# Requirements
+# --- REQUIREMENTS MODIFIED FOR ANNIF ---
+# Annif uses Numpy ~= 2.2.6. We must ensure we build against NumPy 2.x headers.
 numpy_requires = [
-    'numpy>=1.19.5,<2.0.0; python_version>="3.9"'
+    'numpy>=2.0.0; python_version>="3.10"'
 ]
 setup_requires = numpy_requires + [
     'pytest-runner'
 ]
 install_requires = numpy_requires + [
-    'scipy>=1.4.1,<1.14.0',
-    'scikit-learn>=0.24.1',
-    'torch>=2.0; python_version>="3.9"',
-    'sentencepiece>=0.1.86,!=0.1.92', # 0.1.92 results in error for transformers
-    'transformers>=4.31.0; python_version>="3.9"',  # the minimal version supporting py3.9
-    'peft>=0.11.0; python_version>="3.9"',
-    'datasets>=2.19.1; python_version>="3.9"',
+    'scipy>=1.15.3',        # Compatible with Annif's ~=1.15.3
+    'scikit-learn>=1.0',    # Bumped for safety, Annif uses ~=1.7.1
+    'torch>=2.0; python_version>="3.10"',
+    'sentencepiece>=0.1.86,!=0.1.92', 
+    'transformers>=4.49.0; python_version>="3.10"',
+    'peft>=0.11.0; python_version>="3.10"',
+    'datasets>=2.19.1; python_version>="3.10"',
 ]
 
 # Fetch Numpy before building Numpy-dependent extension, if Numpy required version was not installed
@@ -114,10 +115,23 @@ def get_version(cls):
     manual_compile_args = []
 
 # Compile C/C++ extension
+# Ensure include_dirs can find the numpy headers
+try:
+    import numpy
+    np_include = numpy.get_include()
+except ImportError:
+    # Fall back to system includes only; avoid duplicating /usr/include
+    np_include = None
+
+include_dirs = ["pecos/core"]
+if np_include:
+    include_dirs.append(np_include)
+include_dirs.extend(["/usr/include/", "/usr/local/include"])
+
 ext_module = setuptools.Extension(
     "pecos.core.libpecos_float32",
     sources=["pecos/core/libpecos.cpp"],
-    include_dirs=["pecos/core", "/usr/include/", "/usr/local/include"],
+    include_dirs=include_dirs,
     libraries=["gomp", "gcc", "stdc++"],
     extra_compile_args=["-fopenmp", "-O3", "-std=c++17"] + manual_compile_args,
 )