Merge pull request #327 from satra/enh/distlogging

WIP: distributing config and logging options to remote processes

Author: chrisgorgo

CHANGES

@@ -1,7 +1,10 @@
 Changes since 0.5
 =================
 
-None
+* API: Logging to file is disabled by default
+* API: New location of log file -> .nipype/nipype.cfg
+
+* ENH: Changing logging options via config works for distributed processing
 
 Release 0.5 (Mar 10, 2012)
 ==========================

doc/users/config_file.rst

@@ -6,9 +6,9 @@
 
 Some of the system wide options of Nipype can be configured using a
 configuration file. Nipype looks for the file in the local folder under the name
-``nipype.cfg`` and in ``~/.nipype.cfg`` (in this order). If an option will not
-be specified a default value will be assumed. The file is divided into following
-sections:
+``nipype.cfg`` and in ``~/.nipype/nipype.cfg`` (in this order). If an option
+will not be specified a default value will be assumed. The file is divided into
+following sections:
 
 Logging
 ~~~~~~~
@@ -25,7 +25,7 @@ Logging
 	values: ``INFO`` and ``DEBUG``; default value: ``INFO``)
 *log_to_file*
     Indicates whether logging should also send the output to a file (possible
-    values: ``true`` and ``false``; default value: ``true``)
+    values: ``true`` and ``false``; default value: ``false``)
 *log_directory*
 	Where to store logs. (string, default value: home directory)
 *log_size*
@@ -120,43 +120,52 @@ Example
 	hash_method = timestamp
 	display_variable = :1
 
-Additionally you can set some config options by setting the workflow.config. This, however, currently does not work for options related to logging levels. Those will be always read from .cfg files.
-
-Workflow.config property has a form of a nested dictionary reflecting the structure of the .cfg file.
+Workflow.config property has a form of a nested dictionary reflecting the
+structure of the .cfg file.
 
 ::
   
   myworkflow = pe.Workflow()
   myworkflow.config['execution'] = {'stop_on_first_rerun': 'True', 
                                      'hash_method': 'timestamp'}
 
-You can also directly set config options in your workflow script. An
+You can also directly set global config options in your workflow script. An
 example is shown below. This needs to be called before you import the
 pipeline or the logger. Otherwise logging level will not be reset.
 
 ::
 
-  from nipype.utils.config import config
-  from StringIO import StringIO
-  cfg = StringIO("""
-  [logging]
-        workflow_level = DEBUG
+  from nipype import config
+  cfg = dict(logging=dict(workflow_level = 'DEBUG'),
+             execution={'stop_on_first_crash': False,
+                        'hash_method': 'content'})
+  config.update_config(cfg)
 
-  [execution]
-  stop_on_first_crash = false
-  hash_method = content
-  """)
-  
-  config.readfp(cfg)
+Enabling logging to file
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, logging to file is disabled. One can enable and write the file to
+a location of choice as in the example below.
+
+::
+
+    import os
+    from nipype import config, logging
+    config.update_config({'logging': {'log_directory': os.getcwd(),
+                                      'log_to_file': True}})
+    logging.update_logging(config)
+
+The logging update line is necessary to change the behavior of logging such as
+output directory, logging level, etc.,.
 
 Debug configuration
 ~~~~~~~~~~~~~~~~~~~
 
-To enable debug mode, one can insert the following lines at the beginning of any
-script.::
+To enable debug mode, one can insert the following lines::
 
-  from nipype.utils.config import config
+  from nipype import config, logging
   config.enable_debug_mode()
+  logging.update_logging(config)
 
 In this mode the following variables are set::
 

nipype/__init__.py

@@ -6,7 +6,10 @@
                   URL as __url__,
                   STATUS as __status__,
                   __version__)
-from utils.config import config
+from utils.config import NipypeConfig
+config = NipypeConfig()
+from utils.logger import Logging
+logging = Logging(config)
 
 # We require numpy 1.2 for our test suite.  If Tester fails to import,
 # check the version of numpy the user has and inform them they need to

nipype/algorithms/misc.py

@@ -24,28 +24,30 @@
 import scipy.io as sio
 import itertools
 
-from nipype.utils.config import config
+from .. import config, logging
 import matplotlib
 matplotlib.use(config.get("execution", "matplotlib_backend"))
 import matplotlib.pyplot as plt
 
-from nipype.interfaces.base import (BaseInterface, traits, TraitedSpec, File,
-                                    InputMultiPath, OutputMultiPath,
-                                    BaseInterfaceInputSpec, isdefined)
-from nipype.utils.filemanip import fname_presuffix, split_filename
-import logging
-
-logging.basicConfig()
+from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File,
+                               InputMultiPath, OutputMultiPath,
+                               BaseInterfaceInputSpec, isdefined)
+from ..utils.filemanip import fname_presuffix, split_filename
 iflogger = logging.getLogger('interface')
 
 
 class PickAtlasInputSpec(BaseInterfaceInputSpec):
-    atlas = File(exists=True, desc="Location of the atlas that will be used.", mandatory=True)
+    atlas = File(exists=True, desc="Location of the atlas that will be used.",
+                 mandatory=True)
     labels = traits.Either(traits.Int, traits.List(traits.Int),
-                           desc="Labels of regions that will be included in the mask. Must be \
-compatible with the atlas used.", compulsory=True)
-    hemi = traits.Enum('both', 'left', 'right', desc="Restrict the mask to only one hemisphere: left or right", usedefault=True)
-    dilation_size = traits.Int(desc="Defines how much the mask will be dilated (expanded in 3D).", usedefault=True)
+                           desc=("Labels of regions that will be included in"
+                           "the mask. Must be compatible with the atlas used."),
+                           compulsory=True)
+    hemi = traits.Enum('both', 'left', 'right',
+                desc="Restrict the mask to only one hemisphere: left or right",
+                       usedefault=True)
+    dilation_size = traits.Int(usedefault=True,
+            desc="Defines how much the mask will be dilated (expanded in 3D).")
     output_file = File(desc="Where to store the output mask.")
 
 

nipype/algorithms/modelgen.py

@@ -29,9 +29,8 @@
                                     traits, File, Bunch, BaseInterfaceInputSpec,
                                     isdefined)
 from nipype.utils.filemanip import filename_to_list
-from nipype.utils.logger import iflogger
-from nipype.utils.config import config
-
+from .. import config, logging
+iflogger = logging.getLogger('interface')
 
 def gcd(a, b):
     """Returns the greatest common divisor of two integers

nipype/algorithms/rapidart.py

@@ -26,7 +26,7 @@
 from scipy import signal
 import scipy.io as sio
 
-from nipype.utils.config import config
+from .. import config
 import matplotlib
 matplotlib.use(config.get("execution", "matplotlib_backend"))
 import matplotlib.pyplot as plt

nipype/caching/tests/test_memory.py

@@ -8,7 +8,9 @@
 
 from nipype.caching import Memory
 from nipype.pipeline.tests.test_engine import TestInterface
-from nipype.utils.config import config
+from nipype.utils.config import NipypeConfig
+config = NipypeConfig()
+config.set_default_config()
 
 nb_runs = 0
 

nipype/interfaces/ants/base.py

@@ -6,7 +6,7 @@
 from nipype.interfaces.base import (CommandLine, CommandLineInputSpec, traits,
 isdefined)
 
-import logging
+from ... import logging
 logger = logging.getLogger('iflogger')
 
 

nipype/interfaces/base.py

@@ -21,16 +21,16 @@
 from time import time
 from warnings import warn
 
-from nipype.interfaces.traits_extension import (traits, Undefined, TraitDictObject,
-                                                TraitListObject, TraitError,
-                                                isdefined, File, Directory,
-                                                has_metadata)
-from nipype.utils.filemanip import (md5, hash_infile, FileNotFoundError,
-                                    hash_timestamp)
-from nipype.utils.misc import is_container
-from nipype.utils.config import config
-from nipype.utils.logger import iflogger
-from nipype.utils.misc import trim
+from .traits_extension import (traits, Undefined, TraitDictObject,
+                               TraitListObject, TraitError,
+                               isdefined, File, Directory,
+                               has_metadata)
+from ..utils.filemanip import (md5, hash_infile, FileNotFoundError,
+                               hash_timestamp)
+from ..utils.misc import is_container, trim
+from .. import config, logging
+
+iflogger = logging.getLogger('interface')
 
 
 __docformat__ = 'restructuredtext'

nipype/interfaces/cmtk/cmtk.py

@@ -20,9 +20,8 @@
 import nibabel as nb
 import networkx as nx
 import sys
-import logging
 
-logging.basicConfig()
+from ... import logging
 iflogger = logging.getLogger('interface')
 
 def length(xyz, along=False):

nipype/interfaces/cmtk/nx.py

@@ -18,12 +18,10 @@
 import networkx as nx
 import scipy.io as sio
 import pickle
-import logging
 from nipype.utils.misc import package_check
-from nipype.workflows.misc.utils import get_data_dims
 import warnings
 
-logging.basicConfig()
+from ... import logging
 iflogger = logging.getLogger('interface')
 
 have_cmp = True

nipype/interfaces/cmtk/parcellation.py

@@ -18,9 +18,8 @@
 import shutil
 from nipype.utils.misc import package_check
 import warnings
-import logging
 
-logging.basicConfig()
+from ... import logging
 iflogger = logging.getLogger('interface')
 
 have_cmp = True

nipype/interfaces/dipy/tracks.py

@@ -2,14 +2,12 @@
 from nipype.interfaces.base import (TraitedSpec, BaseInterface, BaseInterfaceInputSpec,
                                     File, isdefined, traits)
 from nipype.utils.filemanip import split_filename
-import os, os.path as op
+import os.path as op
 import nibabel as nb, nibabel.trackvis as trk
-import numpy as np
-import logging
 from nipype.utils.misc import package_check
 import warnings
 
-logging.basicConfig()
+from ... import logging
 iflogger = logging.getLogger('interface')
 
 try:

nipype/interfaces/io.py

@@ -38,7 +38,7 @@
 from nipype.utils.filemanip import (copyfile, list_to_filename,
                                     filename_to_list)
 
-import logging
+from .. import logging
 iflogger = logging.getLogger('interface')
 
 

nipype/interfaces/matlab.py

@@ -5,7 +5,7 @@
 
 from nipype.interfaces.base import (CommandLineInputSpec, InputMultiPath, isdefined,
                                     CommandLine, traits, File, Directory)
-from nipype.utils.config import config
+from .. import config
 
 def get_matlab_command():
     if 'NIPYPE_NO_MATLAB' in os.environ:

nipype/interfaces/mrtrix/convert.py

@@ -27,9 +27,7 @@
 except Exception, e:
     warnings.warn('dipy not installed')
 from nibabel.orientations import aff2axcodes
-import logging
-
-logging.basicConfig()
+from ... import logging
 iflogger = logging.getLogger('interface')
 
 def transform_to_affine(streams, header, affine):

nipype/interfaces/mrtrix/tensors.py

@@ -12,12 +12,10 @@
 from nipype.interfaces.base import (CommandLineInputSpec, CommandLine, BaseInterface, BaseInterfaceInputSpec,
                                     traits, File, TraitedSpec, Directory, InputMultiPath, OutputMultiPath, isdefined)
 from nipype.utils.filemanip import split_filename
-import os, os.path as op
+import os.path as op
 import numpy as np
-import nibabel as nb
-import logging
 
-logging.basicConfig()
+from ... import logging
 iflogger = logging.getLogger('interface')
 
 class DWI2SphericalHarmonicsImageInputSpec(CommandLineInputSpec):

nipype/interfaces/spm/base.py

@@ -22,7 +22,7 @@
 
 import nipype.utils.spm_docs as sd
 
-import logging
+from ... import logging
 logger = logging.getLogger('iflogger')
 
 def func_is_3d(in_file):

nipype/interfaces/spm/model.py

@@ -14,7 +14,6 @@
 __docformat__ = 'restructuredtext'
 
 # Standard library imports
-import logging
 import os
 from glob import glob
 
@@ -30,6 +29,7 @@
 from nipype.utils.filemanip import (filename_to_list, list_to_filename,
                                     split_filename)
 
+from ... import logging
 logger = logging.getLogger('spmlogger')
 
 

nipype/interfaces/tests/test_base.py

@@ -5,11 +5,10 @@
 import shutil
 
 from nipype.testing import (assert_equal, assert_not_equal, assert_raises,
-                            assert_true, assert_false, with_setup, package_check, skipif)
+                        assert_true, assert_false, with_setup, package_check,
+                        skipif)
 import nipype.interfaces.base as nib
-from nipype.interfaces.base import Undefined
-
-from nipype.utils.config import config
+from nipype.interfaces.base import Undefined, config
 
 #test Bunch
 def test_bunch():