run lints

Author: CompuIves

.devcontainer/Dockerfile

@@ -3,4 +3,4 @@ FROM mcr.microsoft.com/devcontainers/python:3.9
 ENV PYTHONUNBUFFERED 1
 
 # Install pre-commit
-RUN pip install pre-commit 
+RUN pip install pre-commit

.devcontainer/devcontainer.json

@@ -5,6 +5,7 @@
     },
     "features": {
         "ghcr.io/devcontainers/features/git:1": {},
+        "ghcr.io/devcontainers/features/node:1": {},
         "ghcr.io/devcontainers/features/java:1": {
             "version": "17",
             "installMaven": false,
@@ -37,4 +38,4 @@
     },
     "postCreateCommand": "poetry install",
     "remoteUser": "vscode"
-} 
+}

Makefile

@@ -31,6 +31,10 @@ install:
 format:
 	poetry run pre-commit run --all-files
 
+# OpenAPI Client Generation
+
+generate-client:
+	python scripts/generate_api_client.py
 
 # Documentation
 

examples/tokenize_data.py

@@ -25,9 +25,7 @@ def tokenize_variable_length(
     tokenizer: PreTrainedTokenizerBase,
     add_special_tokens: bool = True,
 ) -> BatchEncoding:
-    tokenized = tokenizer(
-        data["text"], add_special_tokens=add_special_tokens, truncation=False
-    )
+    tokenized = tokenizer(data["text"], add_special_tokens=add_special_tokens, truncation=False)
     return tokenized
 
 
@@ -102,10 +100,7 @@ def pack_sequences(
     output = {"input_ids": packed_sequences}
     if add_labels:
         output["labels"] = [
-            [
-                LOSS_IGNORE_INDEX if token_id == pad_token_id else token_id
-                for token_id in example
-            ]
+            [LOSS_IGNORE_INDEX if token_id == pad_token_id else token_id for token_id in example]
             for example in output["input_ids"]
         ]
 
@@ -201,18 +196,14 @@ def process_data(args: argparse.Namespace) -> None:
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Pretokenize examples for finetuning via Together"
-    )
+    parser = argparse.ArgumentParser(description="Pretokenize examples for finetuning via Together")
     parser.add_argument(
         "--dataset",
         type=str,
         default="clam004/antihallucination_dataset",
         help="Dataset name on the Hugging Face Hub",
     )
-    parser.add_argument(
-        "--max-seq-length", type=int, default=8192, help="Maximum sequence length"
-    )
+    parser.add_argument("--max-seq-length", type=int, default=8192, help="Maximum sequence length")
     parser.add_argument(
         "--add-labels",
         action="store_true",

mypy.ini

@@ -1,2 +1,9 @@
 [mypy]
 plugins = pydantic.mypy
+disallow_untyped_defs = true
+
+[mypy-together.generated.*]
+ignore_errors = true
+
+[mypy.tests.*]
+ignore_errors = true

scripts/.gitignore

@@ -1,2 +1,2 @@
 openapi-generator-cli.jar
-openapi.yaml
+

scripts/generate_api_client.py

@@ -13,7 +13,7 @@
 GENERATOR_JAR = Path(__file__).parent / "openapi-generator-cli.jar"
 
 
-def run_command(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
+def run_command(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess[str]:
     """Run a command and optionally check its return code."""
     print(f"Running: {' '.join(cmd)}")
     return subprocess.run(cmd, check=check, capture_output=True, text=True)
@@ -31,6 +31,9 @@ def main() -> None:
     spec_file = Path(__file__).parent / "openapi.yaml"
     download_file(OPENAPI_SPEC_URL, spec_file)
 
+    # Run formatter on the spec for better merge conflict handling
+    run_command(["npx", "-y", "prettier", "--write", str(spec_file)])
+
     # Download generator if needed
     download_file(GENERATOR_JAR_URL, GENERATOR_JAR)