[common] Add proper val split for incremental loader.

Author: Arthur Douillard

inclearn/lib/data.py

@@ -22,10 +22,17 @@ def __init__(
         workers=10,
         batch_size=128,
         seed=1,
-        increment=10
+        increment=10,
+        validation_split=0.
     ):
         datasets = _get_datasets(dataset_name)
-        self._setup_data(datasets, random_order=random_order, seed=seed, increment=increment)
+        self._setup_data(
+            datasets,
+            random_order=random_order,
+            seed=seed,
+            increment=increment,
+            validation_split=validation_split
+        )
         self.train_transforms = datasets[0].train_transforms  # FIXME handle multiple datasets
         self.common_transforms = datasets[0].common_transforms
 
@@ -48,6 +55,9 @@ def new_task(self, memory=None):
         x_train, y_train = self._select(
             self.data_train, self.targets_train, low_range=min_class, high_range=max_class
         )
+        x_val, y_val = self._select(
+            self.data_val, self.targets_val, low_range=min_class, high_range=max_class
+        )
         x_test, y_test = self._select(self.data_test, self.targets_test, high_range=max_class)
 
         if memory is not None:
@@ -57,6 +67,7 @@ def new_task(self, memory=None):
             y_train = np.concatenate((y_train, targets_memory))
 
         train_loader = self._get_loader(x_train, y_train, mode="train")
+        val_loader = self._get_loader(x_val, y_val, mode="train") if len(x_val) > 0 else None
         test_loader = self._get_loader(x_test, y_test, mode="test")
 
         task_info = {
@@ -71,13 +82,40 @@ def new_task(self, memory=None):
 
         self._current_task += 1
 
-        return task_info, train_loader, test_loader
+        return task_info, train_loader, val_loader, test_loader
+
+    def get_custom_loader(self, class_indexes, mode="test", data_source="train"):
+        """Returns a custom loader.
+
+        :param class_indexes: A list of class indexes that we want.
+        :param mode: Various mode for the transformations applied on it.
+        :param data_source: Whether to fetch from the train, val, or test set.
+        :return: The raw data and a loader.
+        """
+        if not isinstance(class_indexes, list):  # TODO: deprecated, should always give a list
+            class_indexes = [class_indexes]
+
+        if data_source == "train":
+            x, y = self.data_train, self.targets_train
+        elif data_source == "val":
+            x, y = self.data_val, self.targets_val
+        elif data_source == "test":
+            x, y = self.data_test, self.targets_test
+        else:
+            raise ValueError("Unknown data source <{}>.".format(data_source))
 
-    def get_class_loader(self, class_idx, mode="test"):
-        x, y = self._select(
-            self.data_train, self.targets_train, low_range=class_idx, high_range=class_idx + 1
-        )
-        return x, self._get_loader(x, y, shuffle=False, mode=mode)
+        data, targets = [], []
+        for class_index in class_indexes:
+            class_data, class_targets = self._select(
+                x, y, low_range=class_index, high_range=class_index + 1
+            )
+            data.append(class_data)
+            targets.append(class_targets)
+
+        data = np.concatenate(data)
+        targets = np.concatenate(targets)
+
+        return data, self._get_loader(data, targets, shuffle=False, mode=mode)
 
     def _select(self, x, y, low_range=0, high_range=0):
         idxes = np.where(np.logical_and(y >= low_range, y < high_range))[0]
@@ -102,10 +140,11 @@ def _get_loader(self, x, y, shuffle=True, mode="train"):
             num_workers=self._workers
         )
 
-    def _setup_data(self, datasets, random_order=False, seed=1, increment=10):
+    def _setup_data(self, datasets, random_order=False, seed=1, increment=10, validation_split=0.):
         # FIXME: handles online loading of images
         self.data_train, self.targets_train = [], []
         self.data_test, self.targets_test = [], []
+        self.data_val, self.targets_val = [], []
         self.increments = []
         self.class_order = []
 
@@ -115,6 +154,9 @@ def _setup_data(self, datasets, random_order=False, seed=1, increment=10):
             test_dataset = dataset.base_dataset("data", train=False, download=True)
 
             x_train, y_train = train_dataset.data, np.array(train_dataset.targets)
+            x_val, y_val, x_train, y_train = self._split_per_class(
+                x_train, y_train, validation_split
+            )
             x_test, y_test = test_dataset.data, np.array(test_dataset.targets)
 
             order = [i for i in range(len(np.unique(y_train)))]
@@ -126,10 +168,12 @@ def _setup_data(self, datasets, random_order=False, seed=1, increment=10):
 
             self.class_order.append(order)
 
-            y_train = np.array(list(map(lambda x: order.index(x), y_train)))
-            y_test = np.array(list(map(lambda x: order.index(x), y_test)))
+            y_train = self._map_new_class_index(y_train, order)
+            y_val = self._map_new_class_index(y_val, order)
+            y_test = self._map_new_class_index(y_test, order)
 
             y_train += current_class_idx
+            y_val += current_class_idx
             y_test += current_class_idx
 
             current_class_idx += len(order)
@@ -140,14 +184,53 @@ def _setup_data(self, datasets, random_order=False, seed=1, increment=10):
 
             self.data_train.append(x_train)
             self.targets_train.append(y_train)
+            self.data_val.append(x_val)
+            self.targets_val.append(y_val)
             self.data_test.append(x_test)
             self.targets_test.append(y_test)
 
         self.data_train = np.concatenate(self.data_train)
         self.targets_train = np.concatenate(self.targets_train)
+        self.data_val = np.concatenate(self.data_val)
+        self.targets_val = np.concatenate(self.targets_val)
         self.data_test = np.concatenate(self.data_test)
         self.targets_test = np.concatenate(self.targets_test)
 
+    @staticmethod
+    def _map_new_class_index(y, order):
+        """Transforms targets for new class order."""
+        return np.array(list(map(lambda x: order.index(x), y)))
+
+    @staticmethod
+    def _split_per_class(x, y, validation_split=0.):
+        """Splits train data for a subset of validation data.
+
+        Split is done so that each class has a much data.
+        """
+        shuffled_indexes = np.random.permutation(x.shape[0])
+        x = x[shuffled_indexes]
+        y = y[shuffled_indexes]
+
+        x_val, y_val = [], []
+        x_train, y_train = [], []
+
+        for class_id in np.unique(y):
+            class_indexes = np.where(y == class_id)[0]
+            nb_val_elts = int(class_indexes.shape[0] * validation_split)
+
+            val_indexes = class_indexes[:nb_val_elts]
+            train_indexes = class_indexes[nb_val_elts:]
+
+            x_val.append(x[val_indexes])
+            y_val.append(y[val_indexes])
+            x_train.append(x[train_indexes])
+            y_train.append(y[train_indexes])
+
+        x_val, y_val = np.concatenate(x_val), np.concatenate(y_val)
+        x_train, y_train = np.concatenate(x_train), np.concatenate(y_train)
+
+        return x_val, y_val, x_train, y_train
+
 
 class DummyDataset(torch.utils.data.Dataset):
 

inclearn/lib/factory.py

@@ -51,7 +51,8 @@ def get_data(args):
         random_order=args["random_classes"],
         shuffle=True,
         batch_size=args["batch_size"],
-        workers=args["workers"]
+        workers=args["workers"],
+        validation_split=args["validation"]
     )
 
 

inclearn/models/icarl.py

@@ -46,7 +46,7 @@ def __init__(self, args):
         self._clf_loss = F.binary_cross_entropy_with_logits
         self._distil_loss = F.binary_cross_entropy_with_logits
 
-        self._herding_matrix = np.zeros((100, 500))  # FIXME: nb classes
+        self._herding_matrix = []
 
     def eval(self):
         self._network.eval()
@@ -75,21 +75,15 @@ def _train_task(self, train_loader, val_loader):
         print("nb ", len(train_loader.dataset))
 
         for epoch in range(self._n_epochs):
-            _loss = 0.
+            _loss, val_loss = 0., 0.
 
             self._scheduler.step()
 
             prog_bar = tqdm(train_loader)
-            c = 0
-            for inputs, targets in prog_bar:
-                c += 1
+            for i, (inputs, targets) in enumerate(prog_bar, start=1):
                 self._optimizer.zero_grad()
 
-                inputs, targets = inputs.to(self._device), targets.to(self._device)
-                targets = utils.to_onehot(targets, self._n_classes).to(self._device)
-                logits = self._network(inputs)
-
-                loss = self._compute_loss(inputs, logits, targets)
+                loss = self._forward_loss(inputs, targets)
 
                 if not utils._check_loss(loss):
                     import pdb
@@ -100,14 +94,26 @@ def _train_task(self, train_loader, val_loader):
 
                 _loss += loss.item()
 
+                if val_loader is not None and i == len(train_loader):
+                    for inputs, targets in val_loader:
+                        val_loss += self._forward_loss(inputs, targets).item()
+
                 prog_bar.set_description(
-                    "Task {}/{}, Epoch {}/{} => Clf loss: {}".format(
+                    "Task {}/{}, Epoch {}/{} => Clf loss: {}, Val loss: {}".format(
                         self._task + 1, self._n_tasks,
                         epoch + 1, self._n_epochs,
-                        round(_loss / c, 3)
+                        round(_loss / i, 3),
+                        round(val_loss, 3)
                     )
                 )
 
+    def _forward_loss(self, inputs, targets):
+        inputs, targets = inputs.to(self._device), targets.to(self._device)
+        targets = utils.to_onehot(targets, self._n_classes).to(self._device)
+        logits = self._network(inputs)
+
+        return self._compute_loss(inputs, logits, targets)
+
     def _after_task(self, inc_dataset):
         self.build_examplars(inc_dataset)
 
@@ -182,23 +188,24 @@ def _memory_per_class(self):
     # -----------------
 
     def build_examplars(self, inc_dataset):
+        print("Building & updating memory.")
+
         self._data_memory, self._targets_memory = [], []
         self._class_means = np.zeros((100, self._network.features_dim))
 
         for class_idx in range(self._n_classes):
-            inputs, loader = inc_dataset.get_class_loader(class_idx, mode="test")
+            inputs, loader = inc_dataset.get_custom_loader(class_idx, mode="test")
             features, targets = extract_features(
                 self._network, loader
             )
             features_flipped, _ = extract_features(
-                self._network, inc_dataset.get_class_loader(class_idx, mode="flip")[1]
+                self._network, inc_dataset.get_custom_loader(class_idx, mode="flip")[1]
             )
 
             if class_idx >= self._n_classes - self._task_size:
-                print("Finding examplars for", class_idx)
-                self._herding_matrix[class_idx, :] = select_examplars(
+                self._herding_matrix.append(select_examplars(
                     features, self._memory_per_class
-                )
+                ))
 
             examplar_mean, alph = compute_examplar_mean(
                 features, features_flipped, self._herding_matrix[class_idx], self._memory_per_class
@@ -281,13 +288,4 @@ def compute_accuracy(model, loader, class_means):
     sqd = cdist(class_means, features, 'sqeuclidean')
     score_icarl = (-sqd).T
 
-    # Compute the accuracy over the batch
-    stat_icarl = [
-        ll in best
-        for ll, best in zip(targets_.astype('int32'),
-                            np.argsort(score_icarl, axis=1)[:, -1:])
-    ]
-
-    print("stats ", np.average(stat_icarl))
-
     return np.argsort(score_icarl, axis=1)[:, -1], targets_

inclearn/train.py

@@ -36,7 +36,7 @@ def _train(args):
     memory = None
 
     for _ in range(inc_dataset.n_tasks):
-        task_info, train_loader, test_loader = inc_dataset.new_task(memory)
+        task_info, train_loader, val_loader, test_loader = inc_dataset.new_task(memory)
         if task_info["task"] == args["max_task"]:
             break
 
@@ -50,10 +50,10 @@ def _train(args):
         )
 
         model.eval()
-        model.before_task(train_loader, None)
+        model.before_task(train_loader, val_loader)
         print("Train on {}->{}.".format(task_info["min_class"], task_info["max_class"]))
         model.train()
-        model.train_task(train_loader, None)
+        model.train_task(train_loader, val_loader)
         model.eval()
         model.after_task(inc_dataset)