diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 700ae4f..a956c57 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -6,7 +6,7 @@ jobs: Ubuntu-Python-Unit-Test: name: Ubuntu Python Unit Tests strategy: - max-parallel: 4 + max-parallel: 8 fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13", "3.14"] @@ -44,13 +44,20 @@ jobs: container: amazonlinux:2023 steps: + - name: Install required tools for checkout + run: | + yum -y update + yum -y install \ + tar \ + gzip - uses: actions/checkout@v6 - name: Install dependencies run: | yum -y update yum -y groupinstall 'Development Tools' - yum install spal-release -y - yum install openblas-devel -y + yum -y install \ + spal-release \ + openblas-devel - name: Set up Python ${{ matrix.python-version }} via miniconda run: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh; diff --git a/pecos/utils/smat_util.py b/pecos/utils/smat_util.py index ebf32bf..8df8854 100644 --- a/pecos/utils/smat_util.py +++ b/pecos/utils/smat_util.py @@ -14,7 +14,7 @@ import scipy.sparse as smat -def cs_matrix(arg1, mat_type, shape=None, dtype=None, copy=False, check_contents=False): +def cs_matrix(arg1, mat_type, shape=None, copy=None, dtype=None, check_contents=False): """Custom compressed sparse matrix constructor that allows indices and indptr to be stored in different types. Args: @@ -29,8 +29,8 @@ def cs_matrix(arg1, mat_type, shape=None, dtype=None, copy=False, check_contents compressed sparse matrix in mat_type """ (data, indices, indptr) = arg1 - indices_dtype = smat.sputils.get_index_dtype(indices, check_contents=check_contents) - indptr_dtype = smat.sputils.get_index_dtype(indptr, check_contents=check_contents) + indices_dtype = smat.get_index_dtype(indices, check_contents=check_contents) + indptr_dtype = smat.get_index_dtype(indptr, check_contents=check_contents) ret = mat_type(shape, dtype=dtype) # Read matrix dimensions given, if any @@ -51,7 +51,7 @@ def cs_matrix(arg1, mat_type, shape=None, dtype=None, copy=False, check_contents return ret -def csr_matrix(arg1, shape=None, dtype=None, copy=False): +def csr_matrix(arg1, shape=None, dtype=None, copy=None): """Custom csr_matrix constructor that allows indices and indptr to be stored in different types. Args: @@ -66,7 +66,7 @@ def csr_matrix(arg1, shape=None, dtype=None, copy=False): return cs_matrix(arg1, smat.csr_matrix, shape=shape, dtype=dtype, copy=copy) -def csc_matrix(arg1, shape=None, dtype=None, copy=False): +def csc_matrix(arg1, shape=None, dtype=None, copy=None): """Custom csc_matrix constructor that allows indices and indptr to be stored in different types. Args: @@ -368,7 +368,7 @@ def vstack_csr(matrices, dtype=None): # infer result dtypes from inputs int32max = np.iinfo(np.int32).max if dtype is None: - dtype = smat.sputils.upcast(*[mat.dtype for mat in matrices]) + dtype = np.result_type(*[mat.dtype for mat in matrices]) indices_dtype = np.int64 if nr_cols > int32max else np.int32 indptr_dtype = np.int64 if total_nnz > int32max else np.int32 @@ -417,7 +417,7 @@ def hstack_csr(matrices, dtype=None): # infer result dtypes from inputs int32max = np.iinfo(np.int32).max if dtype is None: - dtype = smat.sputils.upcast(*[mat.dtype for mat in matrices]) + dtype = np.result_type(*[mat.dtype for mat in matrices]) indices_dtype = np.int64 if nr_rows > int32max else np.int32 indptr_dtype = np.int64 if total_nnz > int32max else np.int32 @@ -465,7 +465,7 @@ def block_diag_csr(matrices, dtype=None): # infer result dtypes from inputs int32max = np.iinfo(np.int32).max if dtype is None: - dtype = smat.sputils.upcast(*[mat.dtype for mat in matrices]) + dtype = np.result_type(*[mat.dtype for mat in matrices]) indices_dtype = np.int64 if total_rows > int32max else np.int32 indptr_dtype = np.int64 if total_nnz > int32max else np.int32 diff --git a/pecos/xmc/xtransformer/matcher.py b/pecos/xmc/xtransformer/matcher.py index 23309e9..4b1909c 100644 --- a/pecos/xmc/xtransformer/matcher.py +++ b/pecos/xmc/xtransformer/matcher.py @@ -26,7 +26,8 @@ from pecos.xmc import MLModel, MLProblem, PostProcessor from sklearn.preprocessing import normalize as sk_normalize from torch.utils.data import DataLoader, RandomSampler, SequentialSampler -from transformers import AdamW, AutoConfig, get_scheduler, BatchEncoding +from torch.optim import AdamW +from transformers import AutoConfig, get_scheduler, BatchEncoding from .module import XMCLabelTensorizer, XMCTextTensorizer, XMCTextDataset from .network import ENCODER_CLASSES, HingeLoss, TransformerLinearXMCHead @@ -520,8 +521,8 @@ def text_to_tensor(self, corpus, max_length=None): os.environ["TOKENIZERS_PARALLELISM"] = "true" LOGGER.info("***** Encoding data len={} truncation={}*****".format(len(corpus), max_length)) t_start = time.time() - feature_tensors = self.text_tokenizer.batch_encode_plus( - batch_text_or_text_pairs=corpus, + feature_tensors = self.text_tokenizer( + corpus, **self._get_tokenizer_config(max_length=max_length), ) os.environ["TOKENIZERS_PARALLELISM"] = "false" @@ -811,7 +812,7 @@ def _predict( cpred_csr = smat_util.sorted_csr(cpred_csr, only_topk=local_topk) batch_cpred.append(cpred_csr) else: - cur_act_labels = csr_codes_next[inputs["instance_number"].cpu()] + cur_act_labels = csr_codes_next[inputs["instance_number"].cpu().tolist()] nnz_of_insts = cur_act_labels.indptr[1:] - cur_act_labels.indptr[:-1] inst_idx = np.repeat( np.arange(cur_batch_size, dtype=np.uint32), nnz_of_insts diff --git a/pecos/xmc/xtransformer/network.py b/pecos/xmc/xtransformer/network.py index 5f6ea4b..74e49de 100644 --- a/pecos/xmc/xtransformer/network.py +++ b/pecos/xmc/xtransformer/network.py @@ -32,24 +32,10 @@ DistilBertTokenizerFast, DistilBertPreTrainedModel, ) -from transformers.file_utils import add_start_docstrings -from transformers.modeling_utils import SequenceSummary - -from transformers.models.bert.modeling_bert import BERT_INPUTS_DOCSTRING, BERT_START_DOCSTRING -from transformers.models.roberta.modeling_roberta import ( - RobertaPreTrainedModel, - ROBERTA_INPUTS_DOCSTRING, - ROBERTA_START_DOCSTRING, -) -from transformers.models.xlm_roberta.modeling_xlm_roberta import XLM_ROBERTA_START_DOCSTRING -from transformers.models.xlnet.modeling_xlnet import ( - XLNET_INPUTS_DOCSTRING, - XLNET_START_DOCSTRING, -) -from transformers.models.distilbert.modeling_distilbert import ( - DISTILBERT_INPUTS_DOCSTRING, - DISTILBERT_START_DOCSTRING, -) + +from transformers.utils import auto_docstring +from transformers.models.xlnet.modeling_xlnet import XLNetSequenceSummary as SequenceSummary +from transformers.models.roberta.modeling_roberta import RobertaPreTrainedModel class TransformerModelClass(object): @@ -218,10 +204,7 @@ def forward(self, pooled_output=None, output_indices=None, num_device=1): return W_act, b_act -@add_start_docstrings( - """Bert Model with mutli-label classification head on top for XMC.\n""", - BERT_START_DOCSTRING, -) +@auto_docstring(custom_intro="""Bert Model with mutli-label classification head on top for XMC.\n""") class BertForXMC(BertPreTrainedModel): """ Examples: @@ -239,12 +222,13 @@ def __init__(self, config): self.bert = BertModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.post_init() self.init_weights() def init_from(self, model): self.bert = model.bert - @add_start_docstrings(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) + @auto_docstring def forward( self, input_ids=None, @@ -256,6 +240,12 @@ def forward( label_embedding=None, ): r""" + Args: + label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*): + Pre-computed label embeddings for the multi-label classification head. + head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1. + Returns: :obj:`dict` containing: {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label, @@ -289,10 +279,7 @@ def forward( } -@add_start_docstrings( - """Roberta Model with mutli-label classification head on top for XMC.\n""", - ROBERTA_START_DOCSTRING, -) +@auto_docstring(custom_intro="""Roberta Model with mutli-label classification head on top for XMC.\n""") class RobertaForXMC(RobertaPreTrainedModel): """ Examples: @@ -310,12 +297,13 @@ def __init__(self, config): self.roberta = RobertaModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.post_init() self.init_weights() def init_from(self, model): self.roberta = model.roberta - @add_start_docstrings(ROBERTA_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) + @auto_docstring def forward( self, input_ids=None, @@ -327,6 +315,12 @@ def forward( label_embedding=None, ): r""" + Args: + label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*): + Pre-computed label embeddings for the multi-label classification head. + head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1. + Returns: :obj:`dict` containing: {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label, @@ -361,10 +355,7 @@ def forward( } -@add_start_docstrings( - """XLM-Roberta Model with mutli-label classification head on top for XMC.\n""", - XLM_ROBERTA_START_DOCSTRING, -) +@auto_docstring(custom_intro="""XLM-Roberta Model with mutli-label classification head on top for XMC.\n""") class XLMRobertaForXMC(RobertaForXMC): """ This class overrides :class:`RobertaForXMC`. Please check the superclass for the appropriate @@ -374,10 +365,7 @@ class XLMRobertaForXMC(RobertaForXMC): config_class = XLMRobertaConfig # type: ignore -@add_start_docstrings( - """XLNet Model with mutli-label classification head on top for XMC.\n""", - XLNET_START_DOCSTRING, -) +@auto_docstring(custom_intro="""XLNet Model with mutli-label classification head on top for XMC.\n""") class XLNetForXMC(XLNetPreTrainedModel): """ Examples: @@ -395,12 +383,13 @@ def __init__(self, config): self.transformer = XLNetModel(config) self.sequence_summary = SequenceSummary(config) + self.post_init() self.init_weights() def init_from(self, model): self.transformer = model.transformer - @add_start_docstrings(XLNET_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) + @auto_docstring def forward( self, input_ids=None, @@ -415,6 +404,21 @@ def forward( label_embedding=None, ): r""" + Args: + mems (`List[torch.FloatTensor]`, *optional*): + Pre-computed hidden-states (key and value tensors) of the self-attention blocks. + Can be used to speed up sequential decoding. The `mems` are returned when `use_mems=True` is passed. + perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): + Permutation mask for the attention heads. Can be used for language modeling. + target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): + Mask to map target tokens to input indices. Required for target prediction in PLM tasks. + input_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices (alternative to attention_mask). + label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*): + Pre-computed label embeddings for the multi-label classification head. + head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1. + Returns: :obj:`dict` containing: {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label, @@ -449,10 +453,7 @@ def forward( } -@add_start_docstrings( - """DistilBert Model with mutli-label classification head on top for XMC.\n""", - DISTILBERT_START_DOCSTRING, -) +@auto_docstring(custom_intro="""DistilBert Model with mutli-label classification head on top for XMC.\n""") class DistilBertForXMC(DistilBertPreTrainedModel): """ Examples: @@ -470,12 +471,13 @@ def __init__(self, config): self.distilbert = DistilBertModel(config) self.dropout = nn.Dropout(config.dropout) + self.post_init() self.init_weights() def init_from(self, model): self.distilbert = model.distilbert - @add_start_docstrings(DISTILBERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) + @auto_docstring def forward( self, input_ids=None, @@ -486,6 +488,12 @@ def forward( label_embedding=None, ): r""" + Args: + label_embedding (`torch.FloatTensor` of shape `(num_labels, embedding_dim)`, *optional*): + Pre-computed label embeddings for the multi-label classification head. + head_mask (`torch.FloatTensor` of shape `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values should be either 0 or 1. + Returns: :obj:`dict` containing: {'logits': (:obj:`torch.FloatTensor` of shape (batch_size, num_labels)) pred logits for each label, diff --git a/setup.py b/setup.py index ff5a88b..e7f8d47 100644 --- a/setup.py +++ b/setup.py @@ -84,21 +84,22 @@ def get_version(cls): with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() -# Requirements +# --- REQUIREMENTS MODIFIED FOR ANNIF --- +# Annif uses Numpy ~= 2.2.6. We must ensure we build against NumPy 2.x headers. numpy_requires = [ - 'numpy>=1.19.5,<2.0.0; python_version>="3.9"' + 'numpy>=2.0.0; python_version>="3.10"' ] setup_requires = numpy_requires + [ 'pytest-runner' ] install_requires = numpy_requires + [ - 'scipy>=1.4.1,<1.14.0', - 'scikit-learn>=0.24.1', - 'torch>=2.0; python_version>="3.9"', - 'sentencepiece>=0.1.86,!=0.1.92', # 0.1.92 results in error for transformers - 'transformers>=4.31.0; python_version>="3.9"', # the minimal version supporting py3.9 - 'peft>=0.11.0; python_version>="3.9"', - 'datasets>=2.19.1; python_version>="3.9"', + 'scipy>=1.15.3', # Compatible with Annif's ~=1.15.3 + 'scikit-learn>=1.0', # Bumped for safety, Annif uses ~=1.7.1 + 'torch>=2.0; python_version>="3.10"', + 'sentencepiece>=0.1.86,!=0.1.92', + 'transformers>=4.49.0; python_version>="3.10"', + 'peft>=0.11.0; python_version>="3.10"', + 'datasets>=2.19.1; python_version>="3.10"', ] # Fetch Numpy before building Numpy-dependent extension, if Numpy required version was not installed @@ -114,10 +115,23 @@ def get_version(cls): manual_compile_args = [] # Compile C/C++ extension +# Ensure include_dirs can find the numpy headers +try: + import numpy + np_include = numpy.get_include() +except ImportError: + # Fall back to system includes only; avoid duplicating /usr/include + np_include = None + +include_dirs = ["pecos/core"] +if np_include: + include_dirs.append(np_include) +include_dirs.extend(["/usr/include/", "/usr/local/include"]) + ext_module = setuptools.Extension( "pecos.core.libpecos_float32", sources=["pecos/core/libpecos.cpp"], - include_dirs=["pecos/core", "/usr/include/", "/usr/local/include"], + include_dirs=include_dirs, libraries=["gomp", "gcc", "stdc++"], extra_compile_args=["-fopenmp", "-O3", "-std=c++17"] + manual_compile_args, )