checkSync

#!/usr/bin/env python3
#
# checkSync: compare files that ought to be the same, and report.
# 2015-03-23: Written by Steven J. DeRose.
#
import sys
import os
import re
import subprocess
import codecs

from alogging import ALogger
lg = ALogger()

__metadata__ = {
    "title"        : "checkSync",
    "rightsHolder" : "Steven J. DeRose",
    "creator"      : "http://viaf.org/viaf/50334488",
    "type"         : "http://purl.org/dc/dcmitype/Software",
    "language"     : "Python 3.7",
    "created"      : "2015-03-23",
    "modified"     : "2020-03-04",
    "publisher"    : "http://github.com/sderose",
    "license"      : "https://creativecommons.org/licenses/by-sa/3.0/"
}
__version__ = __metadata__['modified']

descr = """
=Description=

Checks files for whether they're the same, and screams or fixes if not.

==The --filelist option==

Use this option to specify a file, which in turn contains a list of files to
compare. The format:

* Empty lines, or lines beginning with '#', are ignored.

* Lines starting with a name followed by '=' define variables
    MYSVN="/Users/Wayne/Projects/C/"

Such variables can be redefined.

* Other lines should list 2 or more files or directories, separated by
colons (or another character as specified by I<--fieldSep>). Whitespace is
allowed around the field separator, and will be discarded.

When you specify directories rather than lines for checking, they following files
will be excluded from comparison:

* Apparent backup files (~, #, .bak, etc.)

* Hidden files or directories

=Related Commands=

`diff`

=Known bugs and Limitations=

=History=

* 2015-03-23: Written by Steven J. DeRose.

=Rights=

Copyright 2015 by Steven J. DeRose. This work is licensed under a Creative Commons
Attribution-Share Alike 3.0 Unported License. For further information on
this license, see [http://creativecommons.org/licenses/by-sa/3.0].

For the most recent version, see [http://www.derose.net/steve/utilities] or
[http://github.com/sderose].

=Options=
"""

# First col status of index, second status of work tree
gitStats = {  # With -s option
    'M':  'Modified, not committed',
    '?':  'item is not under version control',
    'A':  'Added',
    'D':  'Deleted',
    'R':  'Renamed',
    'C':  'Copied',
    'U':  'Updated but unmerged',
    '!':  'Ignored',
}

svnStats = {
    ' ':  'no modifications',
    'A':  'Added',
    'C':  'Conflicted',
    'D':  'Deleted',
    'I':  'Ignored',
    'M':  'Modified',
    'R':  'Replaced',
    'X':  'an unversioned directory created by an externals definition',
    '?':  'item is not under version control',
    '!':  'item is missing (removed by non-svn command) or incomplete',
    '~':  'versioned item obstructed by some item of a different kind',
}


###############################################################################
#
def processOptions():
    import argparse

    try:
        from BlockFormatter import BlockFormatter
        parser = argparse.ArgumentParser(
            description=descr, formatter_class=BlockFormatter)
    except ImportError:
        parser = argparse.ArgumentParser(description=descr)

    parser.add_argument(
        "--fieldSep",         type=str, default=":",
        help='Paths listed in the --filelist file are separated by this (:).')
    parser.add_argument(
        "--filelist", "-f",   type=str,
        help='Read the list of files to compare, from this file.')
    parser.add_argument(
        "--quiet", "-q",      action='store_true',
        help='Suppress most messages.')
    parser.add_argument(
        "--verbose", "-v",    action='count',       default=0,
        help='Add more messages (repeatable).')
    parser.add_argument(
        "--version",          action='version',     version='Version of '+__version__,
        help='Display version information, then exit.')

    parser.add_argument(
        'files',             type=str,
        nargs=argparse.REMAINDER,
        help='Path(s) to input file(s)')

    args0 = parser.parse_args()
    lg.setVerbose(args0.verbose)
    return(args0)


###############################################################################
#
def compareDirs(paths, depth:int=0):
    un = set()
    lists = []
    for i in range(len(paths)):
        lst = os.listdir(paths[i])
        lists.append(lst)
        un = un.union(set(lst))

    ulist = list(un)
    ulist.sort()
    for i in (range(len(ulist))):
        ilist = []
        for j in range(len(paths)):
            if (ulist[i] in lists[j]):
                ilist.append(os.path.join(paths[j],lists[j]))
            else:
                ilist.append(None)
        compareFiles(ilist, depth=depth)


def compareFiles(paths, depth:int=0):
    ind = "    " * depth
    lg.info(ind + "Checking: '%s'" % (os.path.split(paths[0])[1]))
    sawDiff = False
    date = []
    size = []

    for i in range(len(paths)):
        msg = ""
        p = paths[i]
        if (not os.path.isfile(p)):
            msg = "Missing"
            sawDiff = True
        else:
            date.append(os.path.getmtime(p))
            size.append(os.path.getsize(p))
            if (i==0):
                msg = "(basis)"
            elif (size[i] != size[0]):
                msg = "Size %8d." % (size[i])
                sawDiff = True
            else:
                rc = subprocess.check_output([ 'diff', '-q', paths[0], p ])
                sawDiff = True
                msg = "diff '%s'" % (rc)
        lg.pline(ind + "%d: %s" % (i, os.path.split(p)[0]), msg)
        try:
            rc = subprocess.check_output([ 'git', 'status', '-s', p ])
            if ("Not a git repository" not in rc):
                if (rc != ""):
                    lg.pline(p, "git: '%s'." % (rc[0]))
                    sawDiff = True
        except subprocess.CalledProcessError:
            lg.error("git fail")

        try:
            rc = subprocess.check_output([ 'svn', 'status', p ], stderr=None)
            if ("is not a working copy" not in rc):
                if (not rc.startswith(" ")):
                    lg.pline(p, "svn: '%s'." % (rc[0:2]))
                    sawDiff = True
        except subprocess.CalledProcessError:
            lg.error("svn fail")
    return(sawDiff)


###############################################################################
#
varDict = {}
def lookup(m):
    varName = m.group(1)
    if (varName in varDict): return(varDict[varName])
    return('$'+varName)

def varSub(s):
    s = re.sub(r'\$(\w+)', lookup, s)
    return(s)


###############################################################################
# Main
#
args = processOptions()
varDict = {}
pathSplitExpr = re.compile(r'\s*'+args.fieldSep+r'\s*')
lg.setOption("plineWidth", 65)

if (args.filelist):
    try:
        fh = codecs.open(args.filelist, mode='r', encoding='utf-8')
    except IOError:
        lg.fatal("Unable to open '%s'" % (args.filelist))
        sys.exit()
    recnum = 0
    while (True):
        rec = fh.readline()
        if (rec == ""): break
        recnum += 1
        rec = rec.strip()
        if (rec.startswith('#')): continue
        if (rec==""): continue
        if (re.match(r'\s*\w+\s*=', rec)):
            mat = re.match(r'\s*(\w+)\s*=\s*(.*)$', rec)
            if (mat):
                v = mat.group(2).strip('"\'')
                lg.info("Assigning %s = '%s'" % (mat.group(1), v))
                varDict[mat.group(1)] = varSub(v)
            else:
                lg.error("%s:%d: Bad variable assignment: %s" %
                    (args.filelist, recnum, rec))
        else:
            paths0 = re.split(pathSplitExpr, rec)
            if (len(paths0) < 2):
                lg.error("%s:%d: Fewer than 2 paths: %s" %
                    (args.filelist, recnum, rec))
                continue
            for i0 in range(len(paths0)):
                paths0[i0] = varSub(paths0[i0])
            if (os.path.isdir(paths0[0])): compareDirs(paths0)
            else: compareFiles(paths0)

elif (len(args.files) > 1):
    lg.error("Must use --file for now....")
else:
    lg.error("No fileS or --file specified....")
    sys.exit()

if (not args.quiet): lg.info("Done.")