From ce4ca51dbca73be656feb9b3e5233633e3c5dec7 Mon Sep 17 00:00:00 2001 From: Alessandro Sordoni Date: Tue, 1 Aug 2023 10:13:36 -0400 Subject: [PATCH] Move in projects (#40) * change callbacks * pbar * T0 works with torch==2.0.1 and pyl==2.0.4 * accum inference outputs * update config for finetuning and fix progress * fix inference in t0 * update reqs * move to projects/mhr * create mhr config * remove old import * update t0-3b args * added scripts * added tests * fix value input for Z tuning * update setuup instructions * Loosen python dependencies. * Fix setup and tests. * Increase test verbosity. Removed dependabot pipeline. * Bump pytorch-lightning from 2.0.4 to 2.0.5 * Fix T0 dataset creation script * use /tmp for the datasets preparation * move dataset scripts inside projects/mhr * move some files around * temporarily move _set_defaults back to mttl/config * remove output folder and it ignore them anywhere in the repo. * removed bb * fix missing files in the setup bundle * move fineture scripts to scripts/finetune * remove pl_zeroshot * Review scripts to use same envvar. Add instructions to readme. Save processed data inside the projects/mhr folder. * Removed hardcoded train_dir * add env var to load storycloze dataset * add STORYCLOZE_DIR to readme --------- Co-authored-by: Alessandro Sordoni Co-authored-by: Lucas Caccia Co-authored-by: matheper --- .github/dependabot.yml | 11 - .github/workflows/tests.yml | 2 +- .gitignore | 6 + README.md | 68 +++++- configs/t0/3b.json | 3 - mttl/config.py | 200 +++++++++--------- mttl/dataloader/t0_dataset_readers.py | 15 +- pl_zeroshot.py | 97 --------- .../mhr/configs}/ni/finetune-xl.json | 0 .../mhr/configs}/ni/finetune.json | 0 {configs => projects/mhr/configs}/ni/ia3.json | 0 .../mhr/configs}/ni/local.json | 0 .../mhr/configs}/ni/lora_p.json | 0 .../mhr/configs}/ni/poly_lora.json | 0 .../mhr/configs}/ni/poly_lora_p.json | 0 .../mhr/configs}/ni/pretrain.json | 0 .../mhr/configs}/ni/t5-large-lm.json | 0 .../mhr/configs}/ni/t5-xl-lm.json | 0 .../mhr/configs}/ni/zeroshot.json | 0 projects/mhr/configs/t0/3b.json | 8 + .../mhr/configs}/t0/anli-r1.json | 0 .../mhr/configs}/t0/anli-r2.json | 0 .../mhr/configs}/t0/anli-r3.json | 0 {configs => projects/mhr/configs}/t0/cb.json | 0 .../mhr/configs}/t0/copa.json | 0 .../mhr/configs}/t0/finetune.json | 0 .../mhr/configs}/t0/h-swag.json | 0 {configs => projects/mhr/configs}/t0/ia3.json | 0 .../mhr/configs}/t0/large.json | 0 .../mhr/configs}/t0/local.json | 2 +- .../mhr/configs}/t0/poly_ia3.json | 0 .../mhr/configs}/t0/poly_lora.json | 0 .../mhr/configs}/t0/pretrain.json | 0 {configs => projects/mhr/configs}/t0/rte.json | 0 .../mhr/configs}/t0/storycloze.json | 0 {configs => projects/mhr/configs}/t0/svd.json | 0 {configs => projects/mhr/configs}/t0/t5l.json | 0 {configs => projects/mhr/configs}/t0/wic.json | 0 .../mhr/configs}/t0/winogrande.json | 0 {configs => projects/mhr/configs}/t0/wsc.json | 0 .../mhr/configs}/t0/xl-lm-adapt.json | 0 .../mhr/configs}/xfit/finetune.json | 0 .../mhr/configs}/xfit/fulltune.json | 0 .../mhr/configs}/xfit/local.json | 2 +- .../mhr/configs}/xfit/lora.json | 0 .../mhr/configs}/xfit/poly_ia3.json | 0 .../mhr/configs}/xfit/poly_lora.json | 0 .../mhr/configs}/xfit/pretrain.json | 0 .../mhr/configs}/xfit/t5-large.json | 0 .../mhr/configs}/xfit/t5-xl-lm.json | 0 projects/mhr/datasets/create_datasets.sh | 25 +++ .../mhr/datasets}/create_ni_dataset.py | 0 .../mhr/datasets}/create_t0_dataset.py | 5 +- pl_finetune.py => projects/mhr/pl_finetune.py | 8 +- .../mhr/pl_finetune_tune.py | 0 pl_train.py => projects/mhr/pl_train.py | 4 +- .../mhr/scripts/eval/Z_tuning_few_shot_t0.sh | 13 ++ projects/mhr/scripts/eval/few_shot_t0.sh | 12 ++ .../mhr/scripts/finetune}/few_shot_t0.sh | 4 +- .../finetune}/few_shot_t0_tm0d_with_poly.sh | 4 +- .../scripts/finetune}/few_shot_t0_with_ia3.sh | 4 +- .../mhr/scripts/pretrain/pretrain_lora.sh | 1 + projects/mhr/scripts/pretrain/pretrain_mhr.sh | 1 + .../mhr/scripts/pretrain/pretrain_poly.sh | 1 + .../mhr/utils/get_metrics.py | 0 .../mhr/utils/get_zero_shot_eval.py | 0 pyproject.toml | 4 +- requirements.txt | 10 +- scripts/create_datasets.sh | 35 --- tests/test_config.py | 36 +++- tests/test_poly.py | 9 +- 71 files changed, 291 insertions(+), 299 deletions(-) delete mode 100644 .github/dependabot.yml delete mode 100644 configs/t0/3b.json delete mode 100644 pl_zeroshot.py rename {configs => projects/mhr/configs}/ni/finetune-xl.json (100%) rename {configs => projects/mhr/configs}/ni/finetune.json (100%) rename {configs => projects/mhr/configs}/ni/ia3.json (100%) rename {configs => projects/mhr/configs}/ni/local.json (100%) rename {configs => projects/mhr/configs}/ni/lora_p.json (100%) rename {configs => projects/mhr/configs}/ni/poly_lora.json (100%) rename {configs => projects/mhr/configs}/ni/poly_lora_p.json (100%) rename {configs => projects/mhr/configs}/ni/pretrain.json (100%) rename {configs => projects/mhr/configs}/ni/t5-large-lm.json (100%) rename {configs => projects/mhr/configs}/ni/t5-xl-lm.json (100%) rename {configs => projects/mhr/configs}/ni/zeroshot.json (100%) create mode 100644 projects/mhr/configs/t0/3b.json rename {configs => projects/mhr/configs}/t0/anli-r1.json (100%) rename {configs => projects/mhr/configs}/t0/anli-r2.json (100%) rename {configs => projects/mhr/configs}/t0/anli-r3.json (100%) rename {configs => projects/mhr/configs}/t0/cb.json (100%) rename {configs => projects/mhr/configs}/t0/copa.json (100%) rename {configs => projects/mhr/configs}/t0/finetune.json (100%) rename {configs => projects/mhr/configs}/t0/h-swag.json (100%) rename {configs => projects/mhr/configs}/t0/ia3.json (100%) rename {configs => projects/mhr/configs}/t0/large.json (100%) rename {configs => projects/mhr/configs}/t0/local.json (79%) rename {configs => projects/mhr/configs}/t0/poly_ia3.json (100%) rename {configs => projects/mhr/configs}/t0/poly_lora.json (100%) rename {configs => projects/mhr/configs}/t0/pretrain.json (100%) rename {configs => projects/mhr/configs}/t0/rte.json (100%) rename {configs => projects/mhr/configs}/t0/storycloze.json (100%) rename {configs => projects/mhr/configs}/t0/svd.json (100%) rename {configs => projects/mhr/configs}/t0/t5l.json (100%) rename {configs => projects/mhr/configs}/t0/wic.json (100%) rename {configs => projects/mhr/configs}/t0/winogrande.json (100%) rename {configs => projects/mhr/configs}/t0/wsc.json (100%) rename {configs => projects/mhr/configs}/t0/xl-lm-adapt.json (100%) rename {configs => projects/mhr/configs}/xfit/finetune.json (100%) rename {configs => projects/mhr/configs}/xfit/fulltune.json (100%) rename {configs => projects/mhr/configs}/xfit/local.json (80%) rename {configs => projects/mhr/configs}/xfit/lora.json (100%) rename {configs => projects/mhr/configs}/xfit/poly_ia3.json (100%) rename {configs => projects/mhr/configs}/xfit/poly_lora.json (100%) rename {configs => projects/mhr/configs}/xfit/pretrain.json (100%) rename {configs => projects/mhr/configs}/xfit/t5-large.json (100%) rename {configs => projects/mhr/configs}/xfit/t5-xl-lm.json (100%) create mode 100644 projects/mhr/datasets/create_datasets.sh rename {misc => projects/mhr/datasets}/create_ni_dataset.py (100%) rename {misc => projects/mhr/datasets}/create_t0_dataset.py (91%) rename pl_finetune.py => projects/mhr/pl_finetune.py (98%) rename pl_finetune_tune.py => projects/mhr/pl_finetune_tune.py (100%) rename pl_train.py => projects/mhr/pl_train.py (98%) create mode 100644 projects/mhr/scripts/eval/Z_tuning_few_shot_t0.sh create mode 100644 projects/mhr/scripts/eval/few_shot_t0.sh rename {scripts => projects/mhr/scripts/finetune}/few_shot_t0.sh (68%) rename {scripts => projects/mhr/scripts/finetune}/few_shot_t0_tm0d_with_poly.sh (73%) rename {scripts => projects/mhr/scripts/finetune}/few_shot_t0_with_ia3.sh (71%) create mode 100644 projects/mhr/scripts/pretrain/pretrain_lora.sh create mode 100644 projects/mhr/scripts/pretrain/pretrain_mhr.sh create mode 100644 projects/mhr/scripts/pretrain/pretrain_poly.sh rename get_metrics.py => projects/mhr/utils/get_metrics.py (100%) rename get_zero_shot_eval.py => projects/mhr/utils/get_zero_shot_eval.py (100%) delete mode 100644 scripts/create_datasets.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 91abb11fd..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9f40826ea..6fc5fd96c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,4 +37,4 @@ jobs: # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest \ No newline at end of file + pytest -vv \ No newline at end of file diff --git a/.gitignore b/.gitignore index ce6a95599..f2a59cdaf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ amulet_*/ wandb .amltconfig .amltignore +cache/ +output/ +**/output +data/ +**/data +.vscode/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index 5adab09cf..80885bc63 100644 --- a/README.md +++ b/README.md @@ -4,33 +4,81 @@ MTTL - Multi-Task Transfer Learning ## Setup -Install Python packages: +MTTL supports `Python 3.8` and `Python 3.9`. It is recommended to create a virtual environment for MTTL using `virtualenv` or `conda`. For example, with `conda`: -`pip install -r requirements.txt` + conda create -n mttl python=3.9 + conda activate mttl -_The package `promptsource` currently requires Python 3.7. Alternative versions require local installations (see their [documentation](https://github.com/bigscience-workshop/promptsource#setup))._ +Install the required Python packages: -Download the datasets: + pip install -e . -`bash scripts/create_datasets.sh` -## Multi-task Pre-training -The general command: +## Multi-Head Adapter Routing -`python pl_train.py -c $CONFIG_FILES -k $KWARGS` +Please ensure that you have navigated to the `projects/mhr` directory before running the Multi-Head Adapter Routing scripts: + + cd projects/mhr + + +### Data Preparation + +Download and prepare the datasets for the experiments using the following script: + + bash datasets/create_datasets.sh + + +### Environment Variables + +Based on your experiments, you may need to export one or more of the following environment variables: + + T0_DATA_DIR: `data/t0_data/processed` if you ran the `create_datasets.sh` + NI_DATA_DIR: `data/ni_data/processed` if you ran the `create_datasets.sh` + XFIT_DATA_DIR: `data/ni_data/processed` if you ran the `create_datasets.sh` + CHECKPOINT_DIR + OUTPUT_DIR + CACHE_DIR + STORYCLOZE_DIR: path to your downloaded `.csv` files. See [the storycloze official website](https://cs.rochester.edu/nlp/rocstories/) + + +### Multi-task Pre-training + +The general command for pre-training a model is: + + python pl_train.py -c $CONFIG_FILES -k $KWARGS Multiple `CONFIG_FILES` can be concatenated as `file1+file2`. To modify defaults, `KWARGS` can be expressed as `key=value`. +You can check [scripts/pretrain](scripts/pretrain) for examples. -## Test Fine-Tuning +### Test Fine-Tuning To perform finetuning for a test task, use the script `pl_finetune.py` -## Hyper-parameter Search for Test Fine-Tuning +### Hyper-parameter Search for Test Fine-Tuning To perform an hyperparameter search for a test task, use the script `pl_finetune_tune.py`. The script will just call the functions in `pl_finetune.py` in a loop. The script itself defines hp ranges for different fine-tuning types. + +### Pre-Configured Scripts + +Alternatively, you can run the pre-configured scripts from the `scripts` folder. For example: + + bash scripts/mhr_pretrain.sh + +### Know Issues +If you run into issues with protoc `TypeError: Descriptors cannot not be created directly.`, you can try to downgrade protobuf to 3.20.*: + + pip install protobuf==3.20.* + + +## Running Tests + + pip install -e ".[test]" + pytest -vv tests + + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/configs/t0/3b.json b/configs/t0/3b.json deleted file mode 100644 index e0fcd3b54..000000000 --- a/configs/t0/3b.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "model": "bigscience/T0_3B", -} diff --git a/mttl/config.py b/mttl/config.py index da748dbbf..2916bf15c 100644 --- a/mttl/config.py +++ b/mttl/config.py @@ -6,16 +6,113 @@ class Config: + def __init__(self, filenames=None, kwargs=None, raise_error=True): # Stores personalization of the config file in a dict (json serializable) self._updated_kwargs = {} + self.filenames = filenames + self._set_defaults() + + if filenames: + for filename in filenames.split("+"): + if not os.path.exists(filename): + filename = os.path.join(os.getenv("CONFIG_PATH", default="configs"), filename) + + self.update_kwargs(json.load(open(filename)), eval=False, raise_error=raise_error) + + if kwargs: + self.update_kwargs(kwargs, raise_error=raise_error) + + self.save_config(self.output_dir) + + def was_overridden(self, key): + return key in self._updated_kwargs + + def was_default(self, key): + return key not in self._updated_kwargs + + def update_kwargs(self, kwargs, eval=True, raise_error=True): + for (k, v) in kwargs.items(): + if eval: + try: + v = ast.literal_eval(v) + except (ValueError, SyntaxError): + v = v + else: + v = v + if not hasattr(self, k) and raise_error: + raise ValueError(f"{k} is not in the config") + + if eval: + print("Overwriting {} to {}".format(k, v)) + + if k == 'finegrained': + k = 'poly_granularity' + v = 'finegrained' if v else 'coarsegrained' + elif k in ['train_dir', 'output_dir']: + # this raises an error if the env. var does not exist + v = Template(v).substitute(os.environ) + + setattr(self, k, v) + self._updated_kwargs[k] = v + + def __getitem__(self, item): + return getattr(self, item, None) + + def to_json(self): + """ + Converts parameter values in config to json + :return: json + """ + import copy + + to_save = copy.deepcopy(self.__dict__) + to_save.pop("_updated_kwargs") + + return json.dumps(to_save, indent=4, sort_keys=False) + + def save_config(self, output_dir): + """ + Saves the config + """ + os.makedirs(output_dir, exist_ok=True) + + with open(os.path.join(output_dir, "config.json"), "w+") as fout: + fout.write(self.to_json()) + fout.write("\n") + + @classmethod + def parse(cls, extra_kwargs=None, raise_error=True): + import itertools + + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config_files", required=False) + parser.add_argument("-k", "--kwargs", nargs="*", action='append') + args = parser.parse_args() + + kwargs = {} + if args.kwargs: + kwargs_opts = list(itertools.chain(*args.kwargs)) + for value in kwargs_opts: + key, _, value = value.partition('=') + kwargs[key] = value + args.kwargs = kwargs + if extra_kwargs: + args.kwargs.update(extra_kwargs) + + config = cls(args.config_files, args.kwargs, raise_error=raise_error) + + print(config.to_json()) + return config + + def _set_defaults(self): self.cache_dir = os.getenv("CACHE_DIR", "./cache") self.free_up_space = False # Data config self.dataset = None self.custom_tasks_splits = None - self.train_dir = os.getenv("AMLT_DATA_DIR", "/tmp/") - self.output_dir = os.getenv("AMLT_OUTPUT_DIR", "./output") + self.train_dir = os.getenv("TRAIN_DIR", "/tmp/") + self.output_dir = os.getenv("OUTPUT_DIR", "./output") self.finetune_task_name = None self.example_to_ids_path = None # path to clustering of data self.embeddings_path = None @@ -103,12 +200,12 @@ def __init__(self, filenames=None, kwargs=None, raise_error=True): self.poly_use_shared_skill = False # use one skill shared by all tasks """ - poly_granularity : how granular is the module selection : + poly_granularity : how granular is the module selection : coarsegrained : 1 single selector across all linear layers coderwise : 2 selectors (1 for encoder, 1 for decoder) blockwise : 1 selector for each block of K attention layers (and layernorm) - layerwise : 1 selector for each attention layer (and layernorm) - finegrained : 1 selector for every linear layer + layerwise : 1 selector for each attention layer (and layernorm) + finegrained : 1 selector for every linear layer """ self.poly_granularity = 'finegrained' @@ -119,75 +216,6 @@ def __init__(self, filenames=None, kwargs=None, raise_error=True): self.adapters_weight_decay = None self.module_logits_dropout = 0. self.module_logits_l2_norm = False - self.filenames = filenames - - if filenames: - for filename in filenames.split("+"): - if not os.path.exists(filename): - filename = os.path.join(os.getenv("CONFIG_PATH", default="configs"), filename) - - self.update_kwargs(json.load(open(filename)), eval=False, raise_error=raise_error) - - if kwargs: - self.update_kwargs(kwargs, raise_error=raise_error) - - self.save_config(self.output_dir) - - def was_overridden(self, key): - return key in self._updated_kwargs - - def was_default(self, key): - return key not in self._updated_kwargs - - def update_kwargs(self, kwargs, eval=True, raise_error=True): - for (k, v) in kwargs.items(): - if eval: - try: - v = ast.literal_eval(v) - except (ValueError, SyntaxError): - v = v - else: - v = v - if not hasattr(self, k) and raise_error: - raise ValueError(f"{k} is not in the config") - - if eval: - print("Overwriting {} to {}".format(k, v)) - - if k == 'finegrained': - k = 'poly_granularity' - v = 'finegrained' if v else 'coarsegrained' - elif k in ['train_dir', 'output_dir']: - # this raises an error if the env. var does not exist - v = Template(v).substitute(os.environ) - - setattr(self, k, v) - self._updated_kwargs[k] = v - - def __getitem__(self, item): - return getattr(self, item, None) - - def to_json(self): - """ - Converts parameter values in config to json - :return: json - """ - import copy - - to_save = copy.deepcopy(self.__dict__) - to_save.pop("_updated_kwargs") - - return json.dumps(to_save, indent=4, sort_keys=False) - - def save_config(self, output_dir): - """ - Saves the config - """ - os.makedirs(output_dir, exist_ok=True) - - with open(os.path.join(output_dir, "config.json"), "w+") as fout: - fout.write(self.to_json()) - fout.write("\n") class ParseKwargs(argparse.Action): @@ -196,27 +224,3 @@ def __call__(self, parser, namespace, values, option_string=None): for value in values: key, value = value.split('=') getattr(namespace, self.dest)[key] = value - - -def parse_config(extra_kwargs=None, raise_error=True): - import itertools - - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--config_files", required=False) - parser.add_argument("-k", "--kwargs", nargs="*", action='append') - args = parser.parse_args() - - kwargs = {} - if args.kwargs: - kwargs_opts = list(itertools.chain(*args.kwargs)) - for value in kwargs_opts: - key, _, value = value.partition('=') - kwargs[key] = value - args.kwargs = kwargs - if extra_kwargs: - args.kwargs.update(extra_kwargs) - - config = Config(args.config_files, args.kwargs, raise_error=raise_error) - - print(config.to_json()) - return config diff --git a/mttl/dataloader/t0_dataset_readers.py b/mttl/dataloader/t0_dataset_readers.py index c9827bb65..5f939d980 100644 --- a/mttl/dataloader/t0_dataset_readers.py +++ b/mttl/dataloader/t0_dataset_readers.py @@ -145,7 +145,6 @@ def get_template(self, template_idx): if template_idx >= 0: return self.templates[template_names[template_idx]] elif template_idx == -1: - list_idx = [] list_templates = [] for idx, template_name in enumerate(template_names): @@ -241,7 +240,11 @@ def read_orig_dataset(self, split): else: dataset_stash = ("story_cloze", "2016") orig_data = load_dataset( - *dataset_stash, split=split, cache_dir=self.config.train_dir + *dataset_stash, + split=split, + data_dir=os.environ.get( + "STORYCLOZE_DIR", self.config.train_dir + ), ) orig_data = [example for example in orig_data] for idx, example in enumerate(orig_data): @@ -411,7 +414,9 @@ def __init__(self, config): "BIAS_FAIRNESS": [], } gsheet: Dict[datatset_subset_tuple, Dict] = {} - experiment_path = pkg_resources.resource_filename(__name__, "t0_data/datasets.csv") + experiment_path = pkg_resources.resource_filename( + __name__, "t0_data/datasets.csv" + ) with open(experiment_path) as exp_file: reader = csv.DictReader(exp_file) @@ -505,7 +510,7 @@ def get_task_name(dataset_name, subset_name, template_name): self.t0_base_tasks = [] self.t0_base_templates = [] - for (dataset_name, subset_name, template_name) in added_tasks: + for dataset_name, subset_name, template_name in added_tasks: task_name = get_task_name(dataset_name, subset_name, template_name) if task_name in t0_train_mixture["BASE"]: @@ -544,7 +549,7 @@ def read_orig_dataset(self, split): :param split: split of data """ orig_data = [] - for (dataset_name, subset_name, template_name, cap) in self.t0_base_tasks: + for dataset_name, subset_name, template_name, cap in self.t0_base_tasks: if split == "train": split_num = f"{split}[0:{cap}]" else: diff --git a/pl_zeroshot.py b/pl_zeroshot.py deleted file mode 100644 index e934a79e9..000000000 --- a/pl_zeroshot.py +++ /dev/null @@ -1,97 +0,0 @@ -import copy -import os - -import pytorch_lightning as pl -import torch -from pytorch_lightning import Trainer, seed_everything - -from mttl.callbacks import ProgressCallback -from mttl.config import parse_config -from mttl.datamodule.ni_data_module import NIDataModule -from mttl.datamodule.t0_data_module import T0FinetuneDataModule -from mttl.datamodule.xfit_data_module import XFitDataModule -from mttl.models.encoder_decoder import Finetuner -from mttl.models.t0_encoder_decoder import T0EncoderDecoder -from pl_finetune import ARGS_TO_OVERWRITE -from mttl.utils import get_checkpoint_path, get_mlf_logger - - -def evaluate_zeroshot(config): - seed_everything(config.seed, workers=True) - - if config.checkpoint: - ckpt_path = get_checkpoint_path(config.checkpoint, config.checkpoint_step) - ckpt_beef = torch.load(ckpt_path) - ckpt_args = ckpt_beef["hyper_parameters"] - ckpt_dict = ckpt_beef["state_dict"] - config.old_exp_name = ckpt_args["exp_name"] - - for arg_name in ARGS_TO_OVERWRITE: - if arg_name in ckpt_args and not config.was_overridden(arg_name): - print("Overwriting", arg_name, "=", ckpt_args[arg_name]) - setattr(config, arg_name, ckpt_args[arg_name]) - - kwargs = copy.deepcopy(vars(config)) - kwargs.pop("checkpoint", None) - - # data - if config.dataset == "xfit": - dm = XFitDataModule(config) - model_class = Finetuner - elif config.dataset == "ni": - dm = NIDataModule(config) - model_class = Finetuner - elif config.dataset == "bb": - dm = BBDataModule(config) - model_class = T0EncoderDecoder - elif config.dataset == "t0": - dm = T0FinetuneDataModule(config) - model_class = T0EncoderDecoder - else: - raise NotImplementedError() - - module = model_class(**kwargs, tokenizer=dm.tokenizer) - if config.checkpoint: - print("Loading from checkpoint...", ckpt_path) - module.load_state_dict(ckpt_dict, strict=False) - - # when evaluating polytropon with polytropon routing in zero-shot settings, we need to switch to average - if args.model_modifier and "poly" in args.model_modifier and "poly" in args.poly_selector: - module.model.switch_selector_to_average() - - # legit logging - loggers = [] - if os.environ.get("WANDB_API_KEY"): - wandb_logger = pl.loggers.WandbLogger( - project=config.wandb_project, - name=config.exp_name, - ) - wandb_logger.experiment.save("*.py") - loggers.append(wandb_logger) - else: - wandb_logger = None - - mlf_logger = get_mlf_logger() - if mlf_logger: - loggers.append(mlf_logger) - - trainer = Trainer( - devices=1, - accelerator="gpu", - logger=loggers, - num_sanity_val_steps=0, - default_root_dir=config.output_dir, - log_every_n_steps=5 if config.debug else 50, - strategy=None if not config.compute_strategy else config.compute_strategy, - limit_val_batches=0, - limit_train_batches=0, - callbacks=[ProgressCallback()], - ) - results = trainer.test(module, dm) - print(results) - return results - - -if __name__ == "__main__": - args = parse_config() - evaluate_zeroshot(args) diff --git a/configs/ni/finetune-xl.json b/projects/mhr/configs/ni/finetune-xl.json similarity index 100% rename from configs/ni/finetune-xl.json rename to projects/mhr/configs/ni/finetune-xl.json diff --git a/configs/ni/finetune.json b/projects/mhr/configs/ni/finetune.json similarity index 100% rename from configs/ni/finetune.json rename to projects/mhr/configs/ni/finetune.json diff --git a/configs/ni/ia3.json b/projects/mhr/configs/ni/ia3.json similarity index 100% rename from configs/ni/ia3.json rename to projects/mhr/configs/ni/ia3.json diff --git a/configs/ni/local.json b/projects/mhr/configs/ni/local.json similarity index 100% rename from configs/ni/local.json rename to projects/mhr/configs/ni/local.json diff --git a/configs/ni/lora_p.json b/projects/mhr/configs/ni/lora_p.json similarity index 100% rename from configs/ni/lora_p.json rename to projects/mhr/configs/ni/lora_p.json diff --git a/configs/ni/poly_lora.json b/projects/mhr/configs/ni/poly_lora.json similarity index 100% rename from configs/ni/poly_lora.json rename to projects/mhr/configs/ni/poly_lora.json diff --git a/configs/ni/poly_lora_p.json b/projects/mhr/configs/ni/poly_lora_p.json similarity index 100% rename from configs/ni/poly_lora_p.json rename to projects/mhr/configs/ni/poly_lora_p.json diff --git a/configs/ni/pretrain.json b/projects/mhr/configs/ni/pretrain.json similarity index 100% rename from configs/ni/pretrain.json rename to projects/mhr/configs/ni/pretrain.json diff --git a/configs/ni/t5-large-lm.json b/projects/mhr/configs/ni/t5-large-lm.json similarity index 100% rename from configs/ni/t5-large-lm.json rename to projects/mhr/configs/ni/t5-large-lm.json diff --git a/configs/ni/t5-xl-lm.json b/projects/mhr/configs/ni/t5-xl-lm.json similarity index 100% rename from configs/ni/t5-xl-lm.json rename to projects/mhr/configs/ni/t5-xl-lm.json diff --git a/configs/ni/zeroshot.json b/projects/mhr/configs/ni/zeroshot.json similarity index 100% rename from configs/ni/zeroshot.json rename to projects/mhr/configs/ni/zeroshot.json diff --git a/projects/mhr/configs/t0/3b.json b/projects/mhr/configs/t0/3b.json new file mode 100644 index 000000000..f64914b7a --- /dev/null +++ b/projects/mhr/configs/t0/3b.json @@ -0,0 +1,8 @@ +{ + "model": "bigscience/T0_3B", + "precision": "bf16", + "compute_strategy": "ddp", + "train_batch_size": 8, + "predict_batch_size": 8, + "gradient_accumulation_steps": 2 +} diff --git a/configs/t0/anli-r1.json b/projects/mhr/configs/t0/anli-r1.json similarity index 100% rename from configs/t0/anli-r1.json rename to projects/mhr/configs/t0/anli-r1.json diff --git a/configs/t0/anli-r2.json b/projects/mhr/configs/t0/anli-r2.json similarity index 100% rename from configs/t0/anli-r2.json rename to projects/mhr/configs/t0/anli-r2.json diff --git a/configs/t0/anli-r3.json b/projects/mhr/configs/t0/anli-r3.json similarity index 100% rename from configs/t0/anli-r3.json rename to projects/mhr/configs/t0/anli-r3.json diff --git a/configs/t0/cb.json b/projects/mhr/configs/t0/cb.json similarity index 100% rename from configs/t0/cb.json rename to projects/mhr/configs/t0/cb.json diff --git a/configs/t0/copa.json b/projects/mhr/configs/t0/copa.json similarity index 100% rename from configs/t0/copa.json rename to projects/mhr/configs/t0/copa.json diff --git a/configs/t0/finetune.json b/projects/mhr/configs/t0/finetune.json similarity index 100% rename from configs/t0/finetune.json rename to projects/mhr/configs/t0/finetune.json diff --git a/configs/t0/h-swag.json b/projects/mhr/configs/t0/h-swag.json similarity index 100% rename from configs/t0/h-swag.json rename to projects/mhr/configs/t0/h-swag.json diff --git a/configs/t0/ia3.json b/projects/mhr/configs/t0/ia3.json similarity index 100% rename from configs/t0/ia3.json rename to projects/mhr/configs/t0/ia3.json diff --git a/configs/t0/large.json b/projects/mhr/configs/t0/large.json similarity index 100% rename from configs/t0/large.json rename to projects/mhr/configs/t0/large.json diff --git a/configs/t0/local.json b/projects/mhr/configs/t0/local.json similarity index 79% rename from configs/t0/local.json rename to projects/mhr/configs/t0/local.json index 66aa1622a..75392808f 100644 --- a/configs/t0/local.json +++ b/projects/mhr/configs/t0/local.json @@ -1,6 +1,6 @@ { "dataset": "t0", - "train_dir": "/datadrive2/t-few-data-cache/", + "train_dir": "$T0_DATA_DIR", "precision": 32, "weight_decay": 0, "train_batch_size": 2, diff --git a/configs/t0/poly_ia3.json b/projects/mhr/configs/t0/poly_ia3.json similarity index 100% rename from configs/t0/poly_ia3.json rename to projects/mhr/configs/t0/poly_ia3.json diff --git a/configs/t0/poly_lora.json b/projects/mhr/configs/t0/poly_lora.json similarity index 100% rename from configs/t0/poly_lora.json rename to projects/mhr/configs/t0/poly_lora.json diff --git a/configs/t0/pretrain.json b/projects/mhr/configs/t0/pretrain.json similarity index 100% rename from configs/t0/pretrain.json rename to projects/mhr/configs/t0/pretrain.json diff --git a/configs/t0/rte.json b/projects/mhr/configs/t0/rte.json similarity index 100% rename from configs/t0/rte.json rename to projects/mhr/configs/t0/rte.json diff --git a/configs/t0/storycloze.json b/projects/mhr/configs/t0/storycloze.json similarity index 100% rename from configs/t0/storycloze.json rename to projects/mhr/configs/t0/storycloze.json diff --git a/configs/t0/svd.json b/projects/mhr/configs/t0/svd.json similarity index 100% rename from configs/t0/svd.json rename to projects/mhr/configs/t0/svd.json diff --git a/configs/t0/t5l.json b/projects/mhr/configs/t0/t5l.json similarity index 100% rename from configs/t0/t5l.json rename to projects/mhr/configs/t0/t5l.json diff --git a/configs/t0/wic.json b/projects/mhr/configs/t0/wic.json similarity index 100% rename from configs/t0/wic.json rename to projects/mhr/configs/t0/wic.json diff --git a/configs/t0/winogrande.json b/projects/mhr/configs/t0/winogrande.json similarity index 100% rename from configs/t0/winogrande.json rename to projects/mhr/configs/t0/winogrande.json diff --git a/configs/t0/wsc.json b/projects/mhr/configs/t0/wsc.json similarity index 100% rename from configs/t0/wsc.json rename to projects/mhr/configs/t0/wsc.json diff --git a/configs/t0/xl-lm-adapt.json b/projects/mhr/configs/t0/xl-lm-adapt.json similarity index 100% rename from configs/t0/xl-lm-adapt.json rename to projects/mhr/configs/t0/xl-lm-adapt.json diff --git a/configs/xfit/finetune.json b/projects/mhr/configs/xfit/finetune.json similarity index 100% rename from configs/xfit/finetune.json rename to projects/mhr/configs/xfit/finetune.json diff --git a/configs/xfit/fulltune.json b/projects/mhr/configs/xfit/fulltune.json similarity index 100% rename from configs/xfit/fulltune.json rename to projects/mhr/configs/xfit/fulltune.json diff --git a/configs/xfit/local.json b/projects/mhr/configs/xfit/local.json similarity index 80% rename from configs/xfit/local.json rename to projects/mhr/configs/xfit/local.json index dd07e0732..3c25c72e8 100644 --- a/configs/xfit/local.json +++ b/projects/mhr/configs/xfit/local.json @@ -1,6 +1,6 @@ { "dataset": "xfit", - "train_dir": "/datadrive2/crossfit/data/", + "train_dir": "$XFIT_DATA_DIR", "custom_tasks_splits": "./dataloader/xfit_data/random.json", "precision": 32, "total_steps": 100, diff --git a/configs/xfit/lora.json b/projects/mhr/configs/xfit/lora.json similarity index 100% rename from configs/xfit/lora.json rename to projects/mhr/configs/xfit/lora.json diff --git a/configs/xfit/poly_ia3.json b/projects/mhr/configs/xfit/poly_ia3.json similarity index 100% rename from configs/xfit/poly_ia3.json rename to projects/mhr/configs/xfit/poly_ia3.json diff --git a/configs/xfit/poly_lora.json b/projects/mhr/configs/xfit/poly_lora.json similarity index 100% rename from configs/xfit/poly_lora.json rename to projects/mhr/configs/xfit/poly_lora.json diff --git a/configs/xfit/pretrain.json b/projects/mhr/configs/xfit/pretrain.json similarity index 100% rename from configs/xfit/pretrain.json rename to projects/mhr/configs/xfit/pretrain.json diff --git a/configs/xfit/t5-large.json b/projects/mhr/configs/xfit/t5-large.json similarity index 100% rename from configs/xfit/t5-large.json rename to projects/mhr/configs/xfit/t5-large.json diff --git a/configs/xfit/t5-xl-lm.json b/projects/mhr/configs/xfit/t5-xl-lm.json similarity index 100% rename from configs/xfit/t5-xl-lm.json rename to projects/mhr/configs/xfit/t5-xl-lm.json diff --git a/projects/mhr/datasets/create_datasets.sh b/projects/mhr/datasets/create_datasets.sh new file mode 100644 index 000000000..ec6b64601 --- /dev/null +++ b/projects/mhr/datasets/create_datasets.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +### Natural Instructions ### +git clone https://github.com/allenai/natural-instructions.git /tmp/ni + +for num_examples in 16 32 64 100 1000; do + for seed in 13 42 58; do + echo "Working on seed: $seed" + python datasets/create_ni_dataset.py /tmp/ni --output_path ./data/ni_data/processed/ --seed ${seed} --num_examples ${num_examples} + done +done + +rm -rf /tmp/ni + +### CrossFit ### +git clone https://github.com/INK-USC/CrossFit.git /tmp/xfit +(cd /tmp/xfit/tasks/ && python _build_gym.py --build --n_proc 20) +mkdir -p ./data/xfit_data/processed +mv /tmp/xfit/data ./data/xfit_data/processed +rm -rf /tmp/xfit + +### T0 ### +python datasets/create_t0_dataset.py --output_path ./data/t0_data/processed/ diff --git a/misc/create_ni_dataset.py b/projects/mhr/datasets/create_ni_dataset.py similarity index 100% rename from misc/create_ni_dataset.py rename to projects/mhr/datasets/create_ni_dataset.py diff --git a/misc/create_t0_dataset.py b/projects/mhr/datasets/create_t0_dataset.py similarity index 91% rename from misc/create_t0_dataset.py rename to projects/mhr/datasets/create_t0_dataset.py index 1e3f13622..0604225b6 100644 --- a/misc/create_t0_dataset.py +++ b/projects/mhr/datasets/create_t0_dataset.py @@ -1,8 +1,9 @@ import os + import click -from mttl.config import Config from mttl.datamodule.t0_data_module import T0PretrainDataModule +from mttl.config import Config # T-Few github repo, to get original T0 splits @@ -20,7 +21,7 @@ def main(output_path): # move few-shot test sets to data_path print("Moving few shot sets...") - os.system("cp -rf /tmp/t-few/data/few_shot/ " + output_path + "/few_shot/") + os.system("mv -f /tmp/t-few/data/few_shot/ " + output_path + "/few_shot/") t0_config = Config( filenames=[], diff --git a/pl_finetune.py b/projects/mhr/pl_finetune.py similarity index 98% rename from pl_finetune.py rename to projects/mhr/pl_finetune.py index 420bb8737..3064ab169 100644 --- a/pl_finetune.py +++ b/projects/mhr/pl_finetune.py @@ -3,13 +3,9 @@ import pandas as pd import torch -import math -import numpy as np import pytorch_lightning as pl from pytorch_lightning import Trainer, seed_everything -from pytorch_lightning.callbacks import ModelCheckpoint -from mttl.config import parse_config from mttl.callbacks import ProgressCallback from mttl.datamodule.ni_data_module import NIFinetuneDataModule from mttl.datamodule.xfit_data_module import XFitFinetuneDataModule @@ -18,7 +14,7 @@ from mttl.models.monitors import get_monitors from mttl.models.t0_encoder_decoder import T0EncoderDecoder from mttl.utils import CustomModelCheckpoint, get_checkpoint_path, get_mlf_logger - +from mttl.config import Config # When loading a checkpoint for evaluation, which args from old checkpoint # should overwrite the incoming arguments ? @@ -324,7 +320,7 @@ def finetune_xfit(args, use_mlf=True, do_zs=True): if __name__ == "__main__": - args = parse_config() + args = Config.parse() if args.dataset == "xfit": finetune_xfit(args) diff --git a/pl_finetune_tune.py b/projects/mhr/pl_finetune_tune.py similarity index 100% rename from pl_finetune_tune.py rename to projects/mhr/pl_finetune_tune.py diff --git a/pl_train.py b/projects/mhr/pl_train.py similarity index 98% rename from pl_train.py rename to projects/mhr/pl_train.py index 99a73d672..76dca84af 100644 --- a/pl_train.py +++ b/projects/mhr/pl_train.py @@ -4,7 +4,6 @@ from pytorch_lightning.callbacks import ModelCheckpoint from mttl.online_eval import NIOnlineZeroShot, T0OnlineZeroShot -from mttl.config import parse_config from mttl.callbacks import ProgressCallback from mttl.datamodule.ni_data_module import NIPretrainDataModule from mttl.datamodule.xfit_data_module import XFitPretrainDataModule @@ -13,6 +12,7 @@ from mttl.models.t0_encoder_decoder import T0EncoderDecoder from mttl.models.monitors import get_monitors from mttl.utils import get_mlf_logger +from mttl.config import Config def run_multitask(args): @@ -132,5 +132,5 @@ def run_multitask(args): if __name__ == "__main__": - args = parse_config() + args = Config.parse() run_multitask(args) diff --git a/projects/mhr/scripts/eval/Z_tuning_few_shot_t0.sh b/projects/mhr/scripts/eval/Z_tuning_few_shot_t0.sh new file mode 100644 index 000000000..a5269abc8 --- /dev/null +++ b/projects/mhr/scripts/eval/Z_tuning_few_shot_t0.sh @@ -0,0 +1,13 @@ +# NOTE : must assign a value to `CHECKPOINT_DIR` and `OUTPUT_DIR` +# using this path structure for `output_dir` makes it compatible with `get_metrics.py` + +for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 +do + python -m pl_finetune -c \ + t0/finetune.json+t0/${dataset}.json \ + -k \ + checkpoint=$CHECKPOINT_DIR \ + output_dir=${OUTPUT_DIR}/${dataset} \ + finetune_type=Z \ + $* +done diff --git a/projects/mhr/scripts/eval/few_shot_t0.sh b/projects/mhr/scripts/eval/few_shot_t0.sh new file mode 100644 index 000000000..841cffdfa --- /dev/null +++ b/projects/mhr/scripts/eval/few_shot_t0.sh @@ -0,0 +1,12 @@ +# NOTE : must assign a value to `CHECKPOINT_DIR` and `OUTPUT_DIR` +# using this path structure for `output_dir` makes it compatible with `get_metrics.py` + +for dataset in copa h-swag storycloze winogrande wsc wic rte cb anli-r1 anli-r2 anli-r3 +do + python -m pl_finetune -c \ + t0/finetune.json+t0/${dataset}.json \ + -k \ + checkpoint=$CHECKPOINT_DIR \ + output_dir=${OUTPUT_DIR}/${dataset} \ + $* +done diff --git a/scripts/few_shot_t0.sh b/projects/mhr/scripts/finetune/few_shot_t0.sh similarity index 68% rename from scripts/few_shot_t0.sh rename to projects/mhr/scripts/finetune/few_shot_t0.sh index d0655a896..3e1c38cc3 100644 --- a/scripts/few_shot_t0.sh +++ b/projects/mhr/scripts/finetune/few_shot_t0.sh @@ -4,7 +4,7 @@ do python -m pl_finetune -c \ t0/finetune.json+t0/${dataset}.json \ -k \ - checkpoint=${AMLT_MAP_INPUT_DIR} \ - output_dir=${AMLT_OUTPUT_DIR}/${dataset} \ + checkpoint=${CHECKPOINT_DIR} \ + output_dir=${OUTPUT_DIR}/${dataset} \ $* done diff --git a/scripts/few_shot_t0_tm0d_with_poly.sh b/projects/mhr/scripts/finetune/few_shot_t0_tm0d_with_poly.sh similarity index 73% rename from scripts/few_shot_t0_tm0d_with_poly.sh rename to projects/mhr/scripts/finetune/few_shot_t0_tm0d_with_poly.sh index 3d1f83c1e..cea431960 100644 --- a/scripts/few_shot_t0_tm0d_with_poly.sh +++ b/projects/mhr/scripts/finetune/few_shot_t0_tm0d_with_poly.sh @@ -4,8 +4,8 @@ do python -m pl_finetune -c \ t0/finetune.json+t0/${dataset}.json \ -k \ - checkpoint=${AMLT_MAP_INPUT_DIR} \ - output_dir=${AMLT_OUTPUT_DIR}/${dataset} \ + checkpoint=${CHECKPOINT_DIR} \ + output_dir=${OUTPUT_DIR}/${dataset} \ -k model_modifier=poly_lora \ -k poly_selector=poly done diff --git a/scripts/few_shot_t0_with_ia3.sh b/projects/mhr/scripts/finetune/few_shot_t0_with_ia3.sh similarity index 71% rename from scripts/few_shot_t0_with_ia3.sh rename to projects/mhr/scripts/finetune/few_shot_t0_with_ia3.sh index c3b327e72..2df006402 100644 --- a/scripts/few_shot_t0_with_ia3.sh +++ b/projects/mhr/scripts/finetune/few_shot_t0_with_ia3.sh @@ -4,8 +4,8 @@ do python -m pl_finetune -c \ t0/finetune.json+t0/${dataset}.json+t0/ia3.json \ -k \ - checkpoint=${AMLT_MAP_INPUT_DIR} \ - output_dir=${AMLT_OUTPUT_DIR}/${dataset} \ + checkpoint=${CHECKPOINT_DIR} \ + output_dir=${OUTPUT_DIR}/${dataset} \ finetune_type=A \ $* done diff --git a/projects/mhr/scripts/pretrain/pretrain_lora.sh b/projects/mhr/scripts/pretrain/pretrain_lora.sh new file mode 100644 index 000000000..921dd8a07 --- /dev/null +++ b/projects/mhr/scripts/pretrain/pretrain_lora.sh @@ -0,0 +1 @@ +python pl_train.py -c t0/xl-lm-adapt.json+t0/pretrain.json+t0/poly_lora.json -k n_skills=1 eval_every=20_000 $* \ No newline at end of file diff --git a/projects/mhr/scripts/pretrain/pretrain_mhr.sh b/projects/mhr/scripts/pretrain/pretrain_mhr.sh new file mode 100644 index 000000000..eb48868b1 --- /dev/null +++ b/projects/mhr/scripts/pretrain/pretrain_mhr.sh @@ -0,0 +1 @@ +python pl_train.py -c t0/xl-lm-adapt.json+t0/pretrain.json+t0/poly_lora.json -k n_splits=64 eval_every=20_000 $* \ No newline at end of file diff --git a/projects/mhr/scripts/pretrain/pretrain_poly.sh b/projects/mhr/scripts/pretrain/pretrain_poly.sh new file mode 100644 index 000000000..739eda4e7 --- /dev/null +++ b/projects/mhr/scripts/pretrain/pretrain_poly.sh @@ -0,0 +1 @@ +python pl_train.py -c t0/xl-lm-adapt.json+t0/pretrain.json+t0/poly_lora.json -k eval_every=20_000 $* \ No newline at end of file diff --git a/get_metrics.py b/projects/mhr/utils/get_metrics.py similarity index 100% rename from get_metrics.py rename to projects/mhr/utils/get_metrics.py diff --git a/get_zero_shot_eval.py b/projects/mhr/utils/get_zero_shot_eval.py similarity index 100% rename from get_zero_shot_eval.py rename to projects/mhr/utils/get_zero_shot_eval.py diff --git a/pyproject.toml b/pyproject.toml index 221bf8d32..b81e872dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "mttl" version = "0.0.1" description = "Multi-Task Transfer Learning with Adapters" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.8, <3.10" dynamic = ["dependencies"] classifiers = [ @@ -20,7 +20,7 @@ classifiers = [ dependencies = {file = ["requirements.txt"]} [tool.setuptools] -packages = ["mttl"] +packages = ["mttl", "mttl.dataloader", "mttl.datamodule", "mttl.models"] [project.optional-dependencies] dev = [ diff --git a/requirements.txt b/requirements.txt index 81ee21413..1e9ef4d8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -wandb datasets==2.2.1 transformers==4.19.2 -pytorch-lightning==2.0.4 -wget==3.2 -rouge==1.0.0 -tqdm==4.64.0 +pytorch-lightning==2.0.5 +wandb +rouge +wget +tqdm pandas scikit-learn higher diff --git a/scripts/create_datasets.sh b/scripts/create_datasets.sh deleted file mode 100644 index d1bb73e89..000000000 --- a/scripts/create_datasets.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -set -e - -mkdir -p tmp - -### Natural Instructions ### - -git clone https://github.com/allenai/natural-instructions.git tmp/ni - -for num_examples in 16 32 64 100 1000; do -for seed in 13 42 58; do - echo "Working on seed: $seed" - python misc/create_ni_dataset.py tmp/ni --output_path ./dataloader/ni_data/processed/ --seed ${seed} --num_examples ${num_examples} -done -done - -rm -rf tmp/ni - -### CrossFit ### - -git clone https://github.com/INK-USC/CrossFit.git tmp/xfit - -cd tmp/xfit/tasks/ -python _build_gym.py --build --n_proc 20 -cd .. -mkdir -p ../../dataloader/xfit_data/processed -mv data ../../dataloader/xfit_data/processed -cd ../.. - -rm -rf tmp/xfit - -## T0 ## - -python misc/create_t0_dataset.py --output_path ./dataloader/t0_data/processed/ diff --git a/tests/test_config.py b/tests/test_config.py index 6b03d903b..0b7bc76a4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,25 @@ import json +import pytest + from mttl.config import Config -def test_config_dict_like(tmp_path): +@pytest.fixture +def ConfigTest(tmp_path): + class SimpleConfig(Config): + def _set_defaults(self): + self.train_dir = "train_dir" + self.optimizer = "adafactor" + self.dataset = "t0" + self.model = "t5-small" + self.total_steps = 1000 + self.learning_rate = 1e-3 + self.output_dir = str(tmp_path / "output_dir") + return SimpleConfig + + +def test_config_dict_like(tmp_path, ConfigTest): train_dir = str(tmp_path) optimizer = "adafactor" dataset = "t0" @@ -18,7 +34,7 @@ def test_config_dict_like(tmp_path): "total_steps": total_steps, "learning_rate": learning_rate, } - config = Config(kwargs=config_dict) + config = ConfigTest(kwargs=config_dict) reconstructed_config = json.loads(json.dumps(config.__dict__)) assert optimizer in reconstructed_config["optimizer"] assert dataset in reconstructed_config["dataset"] @@ -28,8 +44,8 @@ def test_config_dict_like(tmp_path): assert learning_rate == reconstructed_config["learning_rate"] -def test_config_was_override_from_kwargs(): - config = Config(kwargs={ +def test_config_was_override_from_kwargs(ConfigTest): + config = ConfigTest(kwargs={ "optimizer": "adafactor", "dataset": "t0", "model": "t5-small", @@ -43,7 +59,7 @@ def test_config_was_override_from_kwargs(): assert config.model == "t5-small" -def test_config_was_override_from_file(tmp_path): +def test_config_was_override_from_file(tmp_path, ConfigTest): config_file = tmp_path / "config.json" config_file.write_text( json.dumps( @@ -54,7 +70,7 @@ def test_config_was_override_from_file(tmp_path): } ) ) - config = Config(filenames=str(config_file)) + config = ConfigTest(filenames=str(config_file)) assert not config.was_overridden("train_dir") assert config.was_overridden("optimizer") assert config.was_overridden("dataset") @@ -64,15 +80,15 @@ def test_config_was_override_from_file(tmp_path): assert config.model == "t5-small" -def test_config_was_default_from_kwargs(): - config = Config(kwargs={"dataset": "t0"}) +def test_config_was_default_from_kwargs(ConfigTest): + config = ConfigTest(kwargs={"dataset": "t0"}) assert not config.was_default("dataset") assert config.was_default("model") -def test_config_was_default_from_file(tmp_path): +def test_config_was_default_from_file(tmp_path, ConfigTest): config_file = tmp_path / "config.json" config_file.write_text(json.dumps({"dataset": "t0"})) - config = Config(filenames=str(config_file)) + config = ConfigTest(filenames=str(config_file)) assert not config.was_default("dataset") assert config.was_default("model") diff --git a/tests/test_poly.py b/tests/test_poly.py index c33972f05..0446fde88 100644 --- a/tests/test_poly.py +++ b/tests/test_poly.py @@ -1,6 +1,5 @@ -import pytest -import torch import os +import torch from pytorch_lightning import seed_everything from transformers import AutoTokenizer from mttl.models.encoder_decoder import EncoderDecoder @@ -9,9 +8,11 @@ def test_poly(tmp_path): os.environ["NI_DATA_DIR"] = str(tmp_path) - os.environ["AMLT_OUTPUT_DIR"] = str(tmp_path / "output") + os.environ["OUTPUT_DIR"] = str(tmp_path / "output") - _args = Config("ni/pretrain.json+ni/poly_lora.json") + _args = Config( + "projects/mhr/configs/ni/pretrain.json+projects/mhr/configs/ni/poly_lora.json", + ) _args.n_tasks = 768 _args.model = "t5-small" _args.warmup_steps = 0