make both accuracy and loss non-mandatory; improve logging

Author: maxim5

hyperengine/impl/tensorflow/tensorflow_runner.py

@@ -24,7 +24,7 @@ def __init__(self, model=None, extra_feed={},
     self._x = self._find_tensor(input)
     self._y = self._find_tensor(label)
     self._mode = self._find_tensor(mode, mandatory=False)
-    self._loss = self._find_tensor(loss)
+    self._loss = self._find_tensor(loss, mandatory=False)
     self._accuracy = self._find_tensor(accuracy, mandatory=False)
     self._minimize = self._find_op(train)
     self._model_size = self._calc_model_size()
@@ -43,12 +43,16 @@ def run_batch(self, batch_x, batch_y):
 
   def evaluate(self, batch_x, batch_y):
     feed_dict = self._get_feed_dict(batch_x, batch_y, 'test')
-    if self._accuracy:
-      loss, accuracy = self._session.run([self._loss, self._accuracy], feed_dict=feed_dict)
-      return {'loss': loss, 'accuracy': accuracy}
-    else:
+    if self._loss is None and self._accuracy is None:
+      return {}
+    if self._accuracy is None:
       loss = self._session.run(self._loss, feed_dict=feed_dict)
       return {'loss': loss}
+    if self._loss is None:
+      accuracy = self._session.run(self._accuracy, feed_dict=feed_dict)
+      return {'accuracy': accuracy}
+    loss, accuracy = self._session.run([self._loss, self._accuracy], feed_dict=feed_dict)
+    return {'loss': loss, 'accuracy': accuracy}
 
   def terminate(self):
     tf.reset_default_graph()
@@ -74,6 +78,7 @@ def _find_tensor(self, name, mandatory=True):
       return self._graph.get_tensor_by_name(name + ':0')
     except KeyError:
       if not mandatory:
+        debug('Tensor not found in Tensorflow graph:', name)
         return None
       warn('Failed to infer a tensor "%s" in Tensorflow graph. '
            'Most likely, you should add "name=\'%s\'" in the placeholder/tensor definition' % (name, name))
@@ -84,6 +89,7 @@ def _find_op(self, name, default=None):
       return self._graph.get_operation_by_name(name)
     except KeyError:
       if default is not None:
+        debug('Op not found in Tensorflow graph:', name)
         return default
       warn('Failed to infer an op "%s" in Tensorflow graph. '
            'Most likely, you should add "name=\'%s\'" in the op definition' % (name, name))

hyperengine/model/base_solver.py

@@ -104,7 +104,7 @@ def _update_epochs_dynamically(self):
       elif args == 2:
         new_epochs = self._dynamic_epochs(curve)
       else:
-        warn('Invalid \"dynamic_epochs\" parameter: '
+        warn('Invalid "dynamic_epochs" parameter: '
              'expected a function with either one or two arguments, but got %s' % args_spec)
 
     if self._epochs != new_epochs:
@@ -115,7 +115,7 @@ def _evaluate_train(self, batch_x, batch_y):
     eval_this_step = self._train_set.step % self._eval_train_every == 0
     if eval_this_step and is_info_logged():
       eval_ = self._runner.evaluate(batch_x, batch_y)
-      self._log_iteration('train_accuracy', eval_.get('loss', 0), eval_.get('accuracy', 0), False)
+      self._log_iteration('train_accuracy', eval_.get('loss'), eval_.get('accuracy'), False)
 
   def _evaluate_validation(self):
     eval_this_step = self._train_set.step % self._eval_validation_every == 0
@@ -127,7 +127,7 @@ def _evaluate_validation(self):
         return
 
       eval_ = self._evaluate(batch_x=self._val_set.x, batch_y=self._val_set.y)
-      self._log_iteration('validation_accuracy', eval_.get('loss', 0), eval_.get('accuracy', 0), True)
+      self._log_iteration('validation_accuracy', eval_.get('loss'), eval_.get('accuracy'), True)
       return eval_
 
   def _evaluate_test(self):
@@ -139,13 +139,27 @@ def _evaluate_test(self):
       return
 
     eval_ = self._evaluate(batch_x=self._test_set.x, batch_y=self._test_set.y)
-    info('Final test_accuracy=%.4f' % eval_.get('accuracy', 0))
+    test_accuracy = eval_.get('accuracy')
+    if test_accuracy is None:
+      warn('Test accuracy evaluation is not available')
+      return
+
+    info('Final test_accuracy=%.4f' % test_accuracy)
     return eval_
 
   def _log_iteration(self, name, loss, accuracy, mark_best):
-    marker = ' *' if mark_best and (accuracy > self._max_val_accuracy) else ''
-    info('Epoch %2d, iteration %7d: loss=%.6f, %s=%.4f%s' %
-         (self._train_set.epochs_completed, self._train_set.index, loss, name, accuracy, marker))
+    message = 'Epoch %2d, iteration %7d' % (self._train_set.epochs_completed, self._train_set.index)
+    if accuracy is not None:
+      marker = ' *' if mark_best and (accuracy > self._max_val_accuracy) else ''
+      if loss is None:
+        info('%s: %s=%.4f%s' % (message, name, accuracy, marker))
+      else:
+        info('%s: loss=%.6f, %s=%.4f%s' % (message, loss, name, accuracy, marker))
+    else:
+      if loss is not None:
+        info('%s: loss=%.6f' % (message, loss))
+      else:
+        info('%s: -- no loss or accuracy defined --' % message)
 
   def _evaluate(self, batch_x, batch_y):
     size = len(batch_x)