code reuse

Author: maxim5

hyperengine/examples/4_1_word2vec_embedding.py

@@ -20,19 +20,10 @@ def __init__(self, **params):
     self._num_skips = params.get('num_skips', 2)
     self._skip_window = params.get('skip_window', 1)
 
-    self._step = 0
-    self._index = 0
-    self._epochs_completed = 0
-    self._just_completed = False
-
     self._vocabulary = None
     self._dictionary = None
     self._data = None
 
-  @property
-  def size(self):
-    return len(self._data)
-
   @property
   def vocabulary(self):
     return self._vocabulary
@@ -41,35 +32,14 @@ def vocabulary(self):
   def vocab_size(self):
     return self._vocab_size
 
-  @property
-  def step(self):
-    return self._step
-
-  @property
-  def index(self):
-    return self._index
-
-  @property
-  def epochs_completed(self):
-    return self._epochs_completed
-
-  @property
-  def just_completed(self):
-    return self._just_completed
-
-  def reset_counters(self):
-    self._step = 0
-    self._index = 0
-    self._epochs_completed = 0
-    self._just_completed = False
-
   def build(self):
     hype.util.debug('Building the data provider')
     words = get_text8('temp-text8/data')
     self._vocabulary = [('UNK', None)] + Counter(words).most_common(self._vocab_size - 1)
     self._vocabulary = np.array([word for word, _ in self._vocabulary])
     self._dictionary = {word: code for code, word in enumerate(self._vocabulary)}
     self._data = np.array([self._dictionary.get(word, 0) for word in words])
+    self._size = len(self._data)
 
     if hype.util.is_debug_logged():
       hype.util.debug('Total words in text: %dM' % (len(words) / 1000000))
@@ -89,7 +59,7 @@ def _generate_batch(self, batch_size, num_skips, skip_window):
     span = 2 * skip_window + 1  # [ skip_window target skip_window ]
     buffer = deque(maxlen=span)
     for _ in range(span):
-      buffer.append(self._data[self._index])
+      buffer.append(self._data[self._index_in_epoch])
       self._inc_index()
     for i in range(batch_size // num_skips):
       target = skip_window  # target label at the center of the buffer
@@ -100,20 +70,10 @@ def _generate_batch(self, batch_size, num_skips, skip_window):
         targets_to_avoid.append(target)
         batch[i * num_skips + j] = buffer[skip_window]
         labels[i * num_skips + j, 0] = buffer[target]
-      buffer.append(self._data[self._index])
+      buffer.append(self._data[self._index_in_epoch])
       self._inc_index()
     return batch, labels
 
-  def _inc_index(self):
-    next = self._index + 1
-    if next >= len(self._data):
-      self._index = 0
-      self._epochs_completed += 1
-      self._just_completed = True
-    else:
-      self._index = next
-      self._just_completed = False
-
 
 def word2vec_model(params):
   # Input data.

hyperengine/model/data_set.py

@@ -11,76 +11,122 @@ class DataProvider(object):
 
 
 class IterableDataProvider(DataProvider):
+  def __init__(self):
+    super(IterableDataProvider, self).__init__()
+    self._size = 0
+    self._step = 0
+    self._epochs_completed = 0
+    self._index_in_epoch = 0
+    self._just_completed = False
+
   @property
   def size(self):
-    raise NotImplementedError
+    """
+    Data size (number of rows)
+    """
+    return self._size
+
+  @property
+  def step(self):
+    """
+    The number of batches processed
+    """
+    return self._step
 
   @property
   def index(self):
-    raise NotImplementedError
+    """
+    Total index of input rows (over all epochs)
+    """
+    return self._epochs_completed * self._size + self._index_in_epoch
+
+  @property
+  def index_in_epoch(self):
+    """
+    The index of input rows in a current epoch
+    """
+    return self._index_in_epoch
 
   @property
   def epochs_completed(self):
-    raise NotImplementedError
+    """
+    A number of completed epochs
+    """
+    return self._epochs_completed
 
   @property
   def just_completed(self):
-    raise NotImplementedError
+    """
+    Whether the previous epoch was just completed
+    """
+    return self._just_completed
 
   def reset_counters(self):
-    raise NotImplementedError
+    """
+    Resets all counters.
+    """
+    self._step = 0
+    self._epochs_completed = 0
+    self._index_in_epoch = 0
+    self._just_completed = False
 
   def next_batch(self, batch_size):
+    """
+    Returns the next `batch_size` examples from this data set.
+    """
     raise NotImplementedError
 
+  def _inc_index(self):
+    index = self._index_in_epoch + 1
+    if index >= self._size:
+      self._index_in_epoch = 0
+      self._epochs_completed += 1
+      self._just_completed = True
+    else:
+      self._index_in_epoch = index
+      self._just_completed = False
+
 
-class DataSet(object):
+class DataSet(IterableDataProvider):
   """
-  A labeled data set. Both examples and labels are stored as numpy arrays.
+  A labeled data set. Both inputs and labels are stored as numpy arrays in memory.
   """
 
   def __init__(self, x, y):
+    super(DataSet, self).__init__()
+
     x = np.array(x)
     y = np.array(y)
     assert x.shape[0] == y.shape[0]
 
-    self.size = x.shape[0]
-    self.x = x
-    self.y = y
-    self.step = 0
-    self.epochs_completed = 0
-    self.index_in_epoch = 0
-    self.just_completed = False
+    self._size = x.shape[0]
+    self._x = x
+    self._y = y
 
   @property
-  def index(self):
-    return self.epochs_completed * self.size + self.index_in_epoch
+  def x(self):
+    return self._x
 
-  def reset_counters(self):
-    self.step = 0
-    self.epochs_completed = 0
-    self.index_in_epoch = 0
-    self.just_completed = False
+  @property
+  def y(self):
+    return self._y
 
   def next_batch(self, batch_size):
-    """
-    Return the next `batch_size` examples from this data set.
-    """
-    if self.just_completed:
-      permutation = np.arange(self.size)
+    if self._just_completed:
+      permutation = np.arange(self._size)
       np.random.shuffle(permutation)
-      self.x = self.x[permutation]
-      self.y = self.y[permutation]
-
-    self.step += 1
-    start = self.index_in_epoch
-    self.index_in_epoch += batch_size
-    end = min(self.index_in_epoch, self.size)
-    if self.index_in_epoch >= self.size:
-      self.index_in_epoch = 0
-    self.just_completed = end == self.size
-    self.epochs_completed += int(self.just_completed)
-    return self.x[start:end], self.y[start:end]
+      self._x = self._x[permutation]
+      self._y = self._y[permutation]
+
+    self._step += 1
+    start = self._index_in_epoch
+    self._index_in_epoch += batch_size
+    end = min(self._index_in_epoch, self._size)
+    if self._index_in_epoch >= self._size:
+      self._index_in_epoch = 0
+    self._just_completed = end == self._size
+    self._epochs_completed += int(self._just_completed)
+    return self._x[start:end], self._y[start:end]
 
 
 def merge_data_sets(ds1, ds2):