Merge branch 'master' into feature/atomic-nibble

Author: betatim

.travis.yml

@@ -12,7 +12,7 @@ notifications:
 # for a PR. So we disable build-on-push for all branches
 # except the ones whitelisted here.
 branches:
-  only: 
+  only:
     - master
 
 matrix:
@@ -69,6 +69,7 @@ script:
 
 # generate all the diagnostic reports
 after_success:
+  - make clean
   - PYTEST_ADDOPTS=-qqq make coverage-gcovr.xml coverage.xml
   # Fix suggested by http://diff-cover.readthedocs.io/en/latest/#troubleshooting
   - git fetch origin master:refs/remotes/origin/master

Makefile

@@ -136,6 +136,7 @@ clean: FORCE
 	rm -f diff-cover.html
 	rm -Rf build dist
 	rm -rf __pycache__/ .eggs/ khmer.egg-info/
+	-rm *.gcov
 
 debug: FORCE
 	export CFLAGS="-pg -fprofile-arcs -D_GLIBCXX_DEBUG_PEDANTIC \

khmer/_khmer.cc

@@ -284,6 +284,8 @@ static bool convert_Pytablesizes_to_vector(PyListObject * sizes_list_o,
 }
 
 
+static FastxParserPtr& _PyObject_to_khmer_ReadParser(PyObject * py_object);
+
 /***********************************************************************/
 
 //
@@ -791,6 +793,16 @@ ReadParser_iter_read_pairs(PyObject * self, PyObject * args )
 }
 
 
+PyObject *
+ReadParser_close(PyObject * self, PyObject * args)
+{
+    FastxParserPtr& rparser = _PyObject_to_khmer_ReadParser(self);
+    rparser->close();
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
 static PyMethodDef _ReadParser_methods [ ] = {
     {
         "iter_reads",       (PyCFunction)ReadParser_iter_reads,
@@ -800,6 +812,10 @@ static PyMethodDef _ReadParser_methods [ ] = {
         "iter_read_pairs",  (PyCFunction)ReadParser_iter_read_pairs,
         METH_VARARGS,       "Iterates over paired reads as pairs."
     },
+    {
+        "close",  (PyCFunction)ReadParser_close,
+        METH_NOARGS,       "Close associated files."
+    },
     { NULL, NULL, 0, NULL } // sentinel
 };
 
@@ -866,7 +882,8 @@ void _init_ReadParser_Type_constants()
 
     // Place pair mode constants into class dictionary.
     int result;
-    PyObject *value = PyLong_FromLong(ReadParser<FastxReader>::PAIR_MODE_IGNORE_UNPAIRED);
+    PyObject *value = PyLong_FromLong(
+                          ReadParser<FastxReader>::PAIR_MODE_IGNORE_UNPAIRED);
     if (value == NULL) {
         Py_DECREF(cls_attrs_DICT);
         return;
@@ -2306,7 +2323,7 @@ CPYCHECKER_TYPE_OBJECT_FOR_TYPEDEF("khmer_KHashtable_Object")
 = {
     PyVarObject_HEAD_INIT(NULL, 0)       /* init & ob_size */
     "_khmer.KHashtable   ",              /*tp_name*/
-    sizeof(khmer_KHashtable_Object)   ,  /*tp_basicsize*/
+    sizeof(khmer_KHashtable_Object),     /*tp_basicsize*/
     0,                                   /*tp_itemsize*/
     0,                                   /*tp_dealloc*/
     0,                                   /*tp_print*/
@@ -2905,7 +2922,7 @@ labelhash_consume_fasta_and_tag_with_labels(khmer_KGraphLabels_Object * me,
     //Py_BEGIN_ALLOW_THREADS
     try {
         hb->consume_fasta_and_tag_with_labels<FastxReader>(filename, total_reads,
-                                                           n_consumed);
+                n_consumed);
     } catch (khmer_file_exception &exc) {
         exc_string = exc.what();
         file_exception = exc_string.c_str();
@@ -3735,8 +3752,9 @@ static PyObject * hllcounter_consume_fasta(khmer_KHLLCounter_Object * me,
     unsigned long long  n_consumed    = 0;
     unsigned int        total_reads   = 0;
     try {
-        me->hllcounter->consume_fasta<FastxReader>(filename, stream_records, total_reads,
-                                      n_consumed);
+        me->hllcounter->consume_fasta<FastxReader>(filename, stream_records,
+                total_reads,
+                n_consumed);
     } catch (khmer_file_exception &exc) {
         PyErr_SetString(PyExc_OSError, exc.what());
         return NULL;

lib/kmer_hash.cc

@@ -184,6 +184,12 @@ HashIntoType _hash_murmur(const std::string& kmer,
     h = out[0];
 
     std::string rev = khmer::_revcomp(kmer);
+    if (rev == kmer) {
+        // self complement kmer, can't use bitwise XOR
+        r = out[0];
+        return h;
+    }
+
     MurmurHash3_x64_128((void *)rev.c_str(), rev.size(), seed, &out);
     r = out[0];
 

lib/read_parsers.cc

@@ -209,11 +209,9 @@ ReadPair ReadParser<SeqIO>::get_next_read_pair(uint8_t mode)
 {
     if (mode == ReadParser<SeqIO>::PAIR_MODE_IGNORE_UNPAIRED) {
         return _get_next_read_pair_in_ignore_mode();
-    }
-    else if (mode == ReadParser<SeqIO>::PAIR_MODE_ERROR_ON_UNPAIRED) {
+    } else if (mode == ReadParser<SeqIO>::PAIR_MODE_ERROR_ON_UNPAIRED) {
         return _get_next_read_pair_in_error_mode();
-    }
-    else {
+    } else {
         std::ostringstream oss;
         oss << "Unknown pair reading mode: " << mode;
         throw UnknownPairReadingMode(oss.str());
@@ -232,6 +230,12 @@ bool ReadParser<SeqIO>::is_complete()
     return _parser->is_complete();
 }
 
+template<typename SeqIO>
+void ReadParser<SeqIO>::close()
+{
+    _parser->close();
+}
+
 void FastxReader::_init()
 {
     seqan::open(_stream, _filename.c_str());
@@ -248,25 +252,25 @@ void FastxReader::_init()
 }
 
 FastxReader::FastxReader()
-        : _filename("-"), _spin_lock(0), _num_reads(0), _have_qualities(false)
+    : _filename("-"), _spin_lock(0), _num_reads(0), _have_qualities(false)
 {
     _init();
 }
 
 FastxReader::FastxReader(const std::string& infile)
-        : _filename(infile),
-          _spin_lock(0),
-          _num_reads(0),
-          _have_qualities(false)
+    : _filename(infile),
+      _spin_lock(0),
+      _num_reads(0),
+      _have_qualities(false)
 {
     _init();
 }
 
 FastxReader::FastxReader(FastxReader& other)
-        : _filename(other._filename),
-          _spin_lock(other._spin_lock),
-          _num_reads(other._num_reads),
-          _have_qualities(other._have_qualities)
+    : _filename(other._filename),
+      _spin_lock(other._spin_lock),
+      _num_reads(other._num_reads),
+      _have_qualities(other._have_qualities)
 {
     _stream = std::move(other._stream);
 }
@@ -286,6 +290,11 @@ size_t FastxReader::get_num_reads()
     return _num_reads;
 }
 
+void FastxReader::close()
+{
+    seqan::close(_stream);
+}
+
 Read FastxReader::get_next_read()
 {
     Read read;
@@ -335,10 +344,10 @@ template<typename SeqIO>
 ReadParserPtr<SeqIO> get_parser(const std::string& filename)
 {
     return ReadParserPtr<SeqIO>(
-        new ReadParser<SeqIO>(
-            std::unique_ptr<SeqIO>(new SeqIO(filename))
-        )
-    );
+               new ReadParser<SeqIO>(
+                   std::unique_ptr<SeqIO>(new SeqIO(filename))
+               )
+           );
 }
 
 // All template instantiations used in the codebase must be declared here.

lib/read_parsers.hh

@@ -93,8 +93,7 @@ struct InvalidReadPair : public  khmer_value_exception {
 unsigned char _to_valid_dna(const unsigned char c);
 
 
-struct Read
-{
+struct Read {
     std::string name;
     std::string description;
     std::string sequence;
@@ -167,6 +166,7 @@ public:
 
     size_t get_num_reads();
     bool is_complete();
+    void close();
 }; // class ReadParser
 
 
@@ -189,6 +189,7 @@ public:
     Read get_next_read();
     bool is_complete();
     size_t get_num_reads();
+    void close();
 }; // class FastxReader
 
 

scripts/trim-low-abund.py

@@ -414,7 +414,8 @@ def main():
         # so pairs will stay together if not orphaned.  This is in contrast
         # to the first loop.  Hence, force_single=True below.
 
-        paired_iter = broken_paired_reader(ReadParser(pass2filename),
+        read_parser = ReadParser(pass2filename)
+        paired_iter = broken_paired_reader(read_parser,
                                            min_length=K,
                                            force_single=True)
 
@@ -432,15 +433,21 @@ def main():
             written_reads += 1
             written_bp += len(read)
 
+        read_parser.close()
+
         log_info('removing {pass2}', pass2=pass2filename)
         os.unlink(pass2filename)
 
         # if we created our own trimfps, close 'em.
         if not args.output:
             trimfp.close()
 
-    log_info('removing temp directory & contents ({temp})', temp=tempdir)
-    shutil.rmtree(tempdir)
+    try:
+        log_info('removing temp directory & contents ({temp})', temp=tempdir)
+        shutil.rmtree(tempdir)
+    except OSError as oe:
+        log_info('WARNING: unable to remove {temp} (probably an NFS issue); '
+                 'please remove manually', temp=tempdir)
 
     trimmed_reads = trimmer.trimmed_reads
 

tests/test_functions.py

@@ -153,6 +153,9 @@ def test_hash_murmur3():
     assert khmer.hash_murmur3('TTTT') == 526240128537019279
     assert khmer.hash_murmur3('CCCC') == 14391997331386449225
     assert khmer.hash_murmur3('GGGG') == 14391997331386449225
+    assert khmer.hash_murmur3('TATATATATATATATATATA') != 0
+    assert khmer.hash_murmur3('TTTTGCAAAA') != 0
+    assert khmer.hash_murmur3('GAAAATTTTC') != 0
 
 
 def test_hash_no_rc_murmur3():