cjjpy.py

# -*- coding: utf-8 -*-

'''
@Author : Jiangjie Chen
@Time   : 2018/11/15 17:08
@Contact: jjchen19@fudan.edu.cn
'''

import re
import datetime
import os
import argparse
import logging
import traceback

try:
    import ujson as json
except:
    import json

HADOOP_BIN = 'PATH=/usr/bin/:$PATH hdfs'
FOR_PUBLIC = True


def LengthStats(filename):
    len_list = []
    thresholds = [0.8, 0.9, 0.95, 0.99, 0.999]
    with open(filename) as f:
        for line in f:
            len_list.append(len(line.strip().split()))
    stats = {
        'Max': max(len_list),
        'Min': min(len_list),
        'Avg': round(sum(len_list) / len(len_list), 4),
    }
    len_list.sort()
    for t in thresholds:
        stats[f"Top-{t}"] = len_list[int(len(len_list) * t)]

    for k in stats:
        print(f"- {k}: {stats[k]}")
    return stats


class AttrDict(dict):
    def __init__(self, *args, **kwargs):
        super(AttrDict, self).__init__(*args, **kwargs)
        self.__dict__ = self


def TraceBack(error_msg):
    exc = traceback.format_exc()
    msg = f'[Error]: {error_msg}.\n[Traceback]: {exc}'
    return msg


def Now():
    return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")


def AbsParentDir(file, parent='..', postfix=None):
    ppath = os.path.abspath(file)
    parent_level = parent.count('.')
    while parent_level > 0:
        ppath = os.path.dirname(ppath)
        parent_level -= 1
    if postfix is not None:
        return os.path.join(ppath, postfix)
    else:
        return ppath


def init_logger(log_file=None, log_file_level=logging.NOTSET, from_scratch=False):
    from coloredlogs import ColoredFormatter
    import tensorflow as tf

    fmt = "[%(asctime)s %(levelname)s] %(message)s"
    log_format = ColoredFormatter(fmt=fmt)
    # log_format = logging.Formatter()
    logger = logging.getLogger()
    logger.setLevel(log_file_level)

    console_handler = logging.StreamHandler()
    console_handler.setFormatter(log_format)
    logger.handlers = [console_handler]

    if log_file and log_file != '':
        if from_scratch and tf.io.gfile.exists(log_file):
            logger.warning('Removing previous log file: %s' % log_file)
            tf.io.gfile.remove(log_file)
        path = os.path.dirname(log_file)
        os.makedirs(path, exist_ok=True)
        file_handler = logging.FileHandler(log_file)
        file_handler.setLevel(log_file_level)
        file_handler.setFormatter(log_format)
        logger.addHandler(file_handler)

    return logger


def OverWriteCjjPy(root='.'):
    # import difflib
    # diff = difflib.HtmlDiff()
    cnt = 0
    golden_cjjpy = os.path.join(root, 'cjjpy.py')
    # golden_content = open(golden_cjjpy).readlines()
    for dir, folder, file in os.walk(root):
        for f in file:
            if f == 'cjjpy.py':
                cjjpy = '%s/%s' % (dir, f)
                # content = open(cjjpy).readlines()
                # d = diff.make_file(golden_content, content)
                cnt += 1
                print('[%d]: %s' % (cnt, cjjpy))
                os.system('cp %s %s' % (golden_cjjpy, cjjpy))


def ChangeFileFormat(filename, new_fmt):
    assert type(filename) is str and type(new_fmt) is str
    spt = filename.split('.')
    if len(spt) == 0:
        return filename
    else:
        return filename.replace('.' + spt[-1], new_fmt)


def CountLines(fname):
    with open(fname, 'rb') as f:
        count = 0
        last_data = '\n'
        while True:
            data = f.read(0x400000)
            if not data:
                break
            count += data.count(b'\n')
            last_data = data
        if last_data[-1:] != b'\n':
            count += 1  # Remove this if a wc-like count is needed
    return count


def GetDate():
    return str(datetime.datetime.now())[5:10].replace('-', '')


def TimeClock(seconds):
    sec = int(seconds)
    hour = int(sec / 3600)
    min = int((sec - hour * 3600) / 60)
    ssec = float(seconds) - hour * 3600 - min * 60
    # return '%dh %dm %.2fs' % (hour, min, ssec)
    return '{0:>2d}h{1:>3d}m{2:>6.2f}s'.format(hour, min, ssec)


def StripAll(text):
    return text.strip().replace('\t', '').replace('\n', '').replace(' ', '')


def GetBracket(text, bracket, en_br=False):
    # input should be aa(bb)cc, True for bracket, False for text
    if bracket:
        try:
            return re.findall('\（(.*?)\）', text.strip())[-1]
        except:
            return ''
    else:
        if en_br:
            text = re.sub('\(.*?\)', '', text.strip())
        return re.sub('（.*?）', '', text.strip())


def CharLang(uchar, lang):
    assert lang.lower() in ['en', 'cn', 'zh']
    if lang.lower() in ['cn', 'zh']:
        if uchar >= '\u4e00' and uchar <= '\u9fa5':
            return True
        else:
            return False
    elif lang.lower() == 'en':
        if (uchar <= 'Z' and uchar >= 'A') or (uchar <= 'z' and uchar >= 'a'):
            return True
        else:
            return False
    else:
        raise NotImplementedError


def WordLang(word, lang):
    for i in word.strip():
        if i.isspace(): continue
        if not CharLang(i, lang):
            return False
    return True


def SortDict(_dict, reverse=True):
    assert type(_dict) is dict
    return sorted(_dict.items(), key=lambda d: d[1], reverse=reverse)


def lark(content='test'):
    print(content)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--diff', nargs=2,
                        help='show difference between two files, shown in downloads/diff.html')
    parser.add_argument('--de_unicode', action='store_true', default=False,
                        help='remove unicode characters')
    parser.add_argument('--link_entity', action='store_true', default=False,
                        help='')
    parser.add_argument('--max_comm_len', action='store_true', default=False,
                        help='')
    parser.add_argument('--search', nargs=2,
                        help='search key from file, 2 args: file name & key')
    parser.add_argument('--email', nargs=2,
                        help='sending emails, 2 args: subject & content')
    parser.add_argument('--overwrite', action='store_true', default=None,
                        help='overwrite all cjjpy under given *dir* based on *dir*/cjjpy.py')
    parser.add_argument('--replace', nargs=3,
                        help='replace char, 3 args: file name & replaced char & replacer char')
    parser.add_argument('--lark', nargs=1)
    parser.add_argument('--get_hdfs', nargs=2,
                        help='easy copy from hdfs to local fs, 2 args: remote_file/dir & local_dir')
    parser.add_argument('--put_hdfs', nargs=2,
                        help='easy put from local fs to hdfs, 2 args: local_file/dir & remote_dir')
    parser.add_argument('--length_stats', nargs=1,
                        help='simple token lengths distribution of a line-by-line file')

    args = parser.parse_args()

    if args.overwrite:
        print('* Overwriting cjjpy...')
        OverWriteCjjPy()

    if args.lark:
        try:
            content = args.lark[0]
        except:
            content = 'running complete'
        print(f'* Larking "{content}"...')
        lark(content)

    if args.length_stats:
        file = args.length_stats[0]
        print(f'* Working on {file} lengths statistics...')
        LengthStats(file)