updates

Author: Programmer-RD-AI

ML/__init__.py

@@ -28,9 +28,11 @@
 from torch.hub import *
 import torchtext.functional as F
 import warnings
+import torch.multiprocessing
 
-warnings.filterwarnings("ignore")
 print(torch.__version__, torchvision.__version__, torchtext.__version__)
+torch.multiprocessing.set_sharing_strategy("file_system")
+warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
 # os.environ["WANDB_SILENT"] = "true"
 PROJECT_NAME = "NLP-Disaster Tweets"

ML/dataset/__init__.py

@@ -1,3 +1,5 @@
+"""Import all of the files in the dataset loading process such as `loader`, `main_loaders`, `valid_loaders`"""
+
 from ML.dataset.loader import *
 from ML.dataset.main_loaders import *
 from ML.dataset.valid_loaders import *

ML/dataset/loader.py

@@ -1,11 +1,21 @@
 from ML import *
 
+"""This file contains the main loader class, which is inherited from in `main_loaders` and `valid_loaders`"""
+
 
 class Loader(Dataset):
-    def __init__(self, path: str, transform: bool = None) -> None:
+    def __init__(self, path: str, transform: torchtext.transforms) -> None:
+        """initalization of the Loader class
+
+        Keyword arguments:
+        path -- path of the .csv file to load
+        transform -- the transformation to be applied to the data
+        Return: None
+        """
         self.path = path
         self.transform = transform
         self.data: pd.DataFrame = pd.read_csv(self.path)
 
     def __len__(self) -> int:
+        """returns the length of the dataset"""
         return len(self.data)

ML/dataset/main_loaders.py

@@ -1,6 +1,8 @@
 from ML import *
 from ML.dataset.loader import *
 
+"""Contains the main dataloader used to load the train and testing data"""
+
 
 class Main_DL(Loader):
     def __init__(
@@ -11,6 +13,15 @@ def __init__(
         batch_size: int = 32,
         **kwargs,
     ) -> None:
+        """initalization of the Main Dataloader which inherits from the `Loader` class
+
+        Keyword arguments:
+        train -- bool, if the data is for training or testing
+        test_split -- float between 0 and 1
+        seed -- int, to prevent change of results
+        batch_size -- int, the size of the batches
+        Return: None
+        """
         super().__init__(**kwargs)
         self.X = self.data["text"].to_numpy()
         self.y = self.data["target"].to_numpy()
@@ -30,6 +41,7 @@ def __init__(
         self.get_batches()
 
     def get_batches(self) -> None:
+        """create the batches for training"""
         X = self.X_train if self.train else self.X_test
         y = self.y_train if self.train else self.y_test
         X_batches = []
@@ -53,6 +65,12 @@ def get_batches(self) -> None:
             self.y_test = np.array(y_batches)
 
     def __getitem__(self, index) -> Tuple[torch.tensor, torch.tensor]:
+        """get an specific item using an specific index
+
+        Keyword arguments:
+        index -- the index of the item to retrieve
+        Return: Tuple
+        """
         if self.train:
             return (
                 self.X_train[index],
@@ -64,4 +82,8 @@ def __getitem__(self, index) -> Tuple[torch.tensor, torch.tensor]:
         )
 
     def __len__(self) -> int:
+        """get the length / no. of batches of the dataset
+
+        Return: Int
+        """
         return len(self.y_train) if self.train else len(self.y_test)

ML/dataset/valid_loaders.py

@@ -1,14 +1,23 @@
 from ML import *
 from ML.dataset.loader import *
 
+"""Contains the validation dataloader used to load the validation"""
+
 
 class Valid_Loader(Loader):
     def __init__(self, *args) -> None:
+        """Initialization of the Valid Loader which inherits from the Loader class"""
         super().__init__(*args)
         self.data["id"].dropna(inplace=True)
         self.X = self.data["text"].to_numpy()
         self.ids = self.data["id"].to_numpy()
         print(len(self.X), len(self.ids))
 
     def __getitem__(self, index) -> np.array:
+        """get and specific item according to the index given
+
+        Keyword arguments:
+        index -- The index of the item
+        Return: Tuple
+        """
         return (self.ids[index], [self.transform(self.X[index])])

ML/helper_functions/__init__.py

@@ -1,4 +1,5 @@
+from ML.helper_functions.clearcache import *
 from ML.helper_functions.load_data import *
 from ML.helper_functions.test import *
 from ML.helper_functions.train import *
-from ML.helper_functions.transformations.transformer import *
+from ML.helper_functions.transformations import *

ML/helper_functions/test.py

@@ -43,7 +43,6 @@ def test(self) -> Dict:
                 f1_tot += f1score
                 l_tot += loss.item()
                 n += 1
-        print(loss.item(), l_tot, l_tot / n)
         return {
             f"{self.name} precision": p_tot / n,
             f"{self.name} recall": r_tot / n,
@@ -55,10 +54,10 @@ def test(self) -> Dict:
     def make_predictions(self, run_name: str, epoch: int) -> pd.DataFrame:
         ids = []
         target = []
-        for i, X in enumerate(self.valid_dataloader):
+        for _id, X in self.valid_dataloader:
             X = F.to_tensor(X, padding_value=1).to("cuda")
             pred = torch.argmax(torch.softmax(self.model(X), dim=1), dim=1).squeeze().cpu().item()
-            ids.append(i)
+            ids.append(_id.item())
             target.append(pred)
         if run_name not in os.listdir("./ML/predictions/"):
             os.mkdir(f"./ML/predictions/{run_name}")

ML/helper_functions/train.py

@@ -72,3 +72,7 @@ def train(self, run_name: str) -> None:
             self.model.train()
         wandb.save()
         wandb.finish()
+        self.save_model(run_name)
+
+    def save_model(self, run_name: str) -> None:
+        pass

run.py

@@ -1,30 +1,38 @@
 from ML import *
 
-train_data_loader, test_data_loader, valid_data_loader = Load_Data(
-    Main_DL,
-    Valid_Loader,
-    [
-        "/media/user/Main/Programmer-RD-AI/Programming/Learning/JS/NLP-Disaster-Tweets/ML/data/train.csv",
-        32,
-        Transformer().transform(),
-    ],
-    [
-        "/media/user/Main/Programmer-RD-AI/Programming/Learning/JS/NLP-Disaster-Tweets/ML/data/test.csv",
-        1,
-    ],
-    0.125,
-    42,
-).ld()
-model = TL().to(device)
-learning_rate = 1e-5
-optimizer = optim.AdamW(model.parameters(), lr=learning_rate)
-criterion = nn.CrossEntropyLoss()
-config = {
-    "model": model,
-    "criterion": criterion,
-    "optimizer": optimizer,
-    "learning_rate": learning_rate,
-}
-Train(
-    model, 10, config, train_data_loader, test_data_loader, valid_data_loader, criterion, optimizer
-).train("wit_randomize")
+lrs = [1e-0, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6]
+for lr in lrs:
+    train_data_loader, test_data_loader, valid_data_loader = Load_Data(
+        Main_DL,
+        Valid_Loader,
+        [
+            "/media/user/Main/Programmer-RD-AI/Programming/Learning/JS/NLP-Disaster-Tweets/ML/data/train.csv",
+            32,
+            Transformer().transform(),
+        ],
+        [
+            "/media/user/Main/Programmer-RD-AI/Programming/Learning/JS/NLP-Disaster-Tweets/ML/data/test.csv",
+            1,
+        ],
+        0.25,
+        42,
+    ).ld()
+    model = TL().to(device)
+    optimizer = optim.Adam(model.parameters(), lr=lr)
+    criterion = nn.CrossEntropyLoss()
+    config = {
+        "model": model,
+        "criterion": criterion,
+        "optimizer": optimizer,
+        "learning_rate": lr,
+    }
+    Train(
+        model,
+        5,
+        config,
+        train_data_loader,
+        test_data_loader,
+        valid_data_loader,
+        criterion,
+        optimizer,
+    ).train(f"{lr}")

test.py

@@ -1,5 +0,0 @@
-from torchvision import transforms
-
-import torch
-
-print(torch.rand(1).item())