Migration to Python 3

issue #812

Author: joke325

Brewfile

@@ -7,5 +7,5 @@ brew "libtool"
 brew "pkg-config"
 brew "gnupg"
 brew "wget"
-brew "python@2"
+brew "python"
 

CMakeLists.txt

@@ -27,7 +27,8 @@
 # 3.10+ for CPackFreeBSD (FreeBSD packaging)
 # 3.10+ for gtest_discover_tests (parallel rnp_tests)
 # 3.12+ for NAMELINK_COMPONENT (for better RPM packaging)
-cmake_minimum_required(VERSION 3.10)
+# 3.12+ for Python3 find module
+cmake_minimum_required(VERSION 3.12)
 
 # contact email, other info
 include(cmake/info.cmake)

ci/before_install.sh

@@ -82,7 +82,7 @@ msys_install() {
     mingw64/mingw-w64-x86_64-cmake
     mingw64/mingw-w64-x86_64-gcc
     mingw64/mingw-w64-x86_64-json-c
-    mingw64/mingw-w64-x86_64-python2
+    mingw64/mingw-w64-x86_64-python3
   "
   pacman --noconfirm -S --needed ${packages}
   pacman --noconfirm -U \

docs/installation.adoc

@@ -93,7 +93,7 @@ pacman -Syu --noconfirm --needed
 # Then most likely you'll need to close msys console and run it agian:
 pacman -Syu --noconfirm --needed
 # Install packages
-pacman --noconfirm -S --needed tar zlib-devel libbz2-devel git automake autoconf libtool automake-wrapper gnupg2 make pkgconfig mingw64/mingw-w64-x86_64-cmake mingw64/mingw-w64-x86_64-gcc mingw64/mingw-w64-x86_64-json-c mingw64/mingw-w64-x86_64-libbotan mingw64/mingw-w64-x86_64-python2
+pacman --noconfirm -S --needed tar zlib-devel libbz2-devel git automake autoconf libtool automake-wrapper gnupg2 make pkgconfig mingw64/mingw-w64-x86_64-cmake mingw64/mingw-w64-x86_64-gcc mingw64/mingw-w64-x86_64-json-c mingw64/mingw-w64-x86_64-libbotan mingw64/mingw-w64-x86_64-python3
 ----
 
 Then clone the repository, say to rnp folder, and:

src/tests/CMakeLists.txt

@@ -28,7 +28,7 @@ find_package(JSON-C 0.11 REQUIRED)
 # Note that we do this call early because Google Test will also do
 # this but with less strict version requirements, which will cause
 # problems for us.
-find_package(PythonInterp 2.7 EXACT)
+find_package(Python3 COMPONENTS Interpreter)
 find_package(GnuPG 2.2 COMPONENTS gpg gpgconf)
 
 include(GoogleTest)
@@ -135,7 +135,7 @@ function(add_cli_test suite)
   set(_test_name cli_tests-${suite})
   add_test(
     NAME ${_test_name}
-    COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -v -d "${suite}"
+    COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -v -d "${suite}"
   )
   set(_env)
   list(APPEND _env
@@ -152,7 +152,7 @@ function(add_cli_test suite)
 endfunction()
 # get a list of test suites
 execute_process(
-  COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -ls
+  COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cli_tests.py" -ls
   RESULT_VARIABLE _ec
   OUTPUT_VARIABLE suitelist
   OUTPUT_STRIP_TRAILING_WHITESPACE

src/tests/cli_common.py

@@ -34,24 +34,22 @@ def raise_err(msg, log = None):
     raise CLIError(msg, log)
 
 def size_to_readable(num, suffix = 'B'):
-    for unit in ['','K','M','G','T','P','E','Z']:
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
         if abs(num) < 1024.0:
             return "%3.1f%s%s" % (num, unit, suffix)
         num /= 1024.0
     return "%.1f%s%s" % (num, 'Yi', suffix)
 
 def list_upto(lst, count):
-    res = lst[:]
-    while len(res) < count:
-        res = res + lst[:]
-    return res[:count]
+    return (list(lst)*(count//len(lst)+1))[:count]
 
 def pswd_pipe(password):
     pr, pw = os.pipe()
     with os.fdopen(pw, 'w') as fw:
         fw.write(password)
         fw.write('\n')
         fw.write(password)
+    os.set_inheritable(pr, True)
 
     if not is_windows():
         return pr
@@ -63,15 +61,16 @@ def pswd_pipe(password):
 def random_text(path, size):
     # Generate random text, with 50% probability good-compressible
     if random.randint(0, 10) < 5:
-        st = ''.join(random.choice(string.ascii_letters + string.digits + " \t\n-,.") for _ in range(size))
+        st = ''.join(random.choice(string.ascii_letters + string.digits + " \t\n-,.")
+                     for _ in range(size))
     else:
         st = ''.join(random.choice("abcdef0123456789 \t\n-,.") for _ in range(size))
     with open(path, 'w+') as f:
         f.write(st)
 
 def file_text(path):
-    with open(path, 'r') as f:
-        return f.read()
+    with open(path, 'rb') as f:
+        return f.read().decode().replace('\r\r', '\r')
 
 def find_utility(name, exitifnone = True):
     path = distutils.spawn.find_executable(name)
@@ -100,17 +99,26 @@ def run_proc_windows(proc, params, stdin=None):
 
     exe = os.path.basename(proc)
     # We need to escape empty parameters/ones with spaces with quotes
-    params = map(lambda st: st if (st and not any(x in st for x in [' ','\r','\t'])) else '"%s"' % st, [exe] + params)
+    params = tuple(map(lambda st: st if (st and not any(x in st for x in [' ','\r','\t'])) else '"%s"' % st, [exe] + params))
     sys.stdout.flush()
 
     stdin_path = os.path.join(WORKDIR, 'stdin.txt')
     stdout_path = os.path.join(WORKDIR, 'stdout.txt')
     stderr_path = os.path.join(WORKDIR, 'stderr.txt')
+    pass_path = os.path.join(WORKDIR, 'pass.txt')
+    passfd = 0
+    passfo = None
+    try:
+        idx = params.index('--pass-fd')
+        if idx < len(params):
+            passfd = int(params[idx+1])
+            passfo = os.fdopen(passfd, 'r', closefd=False)
+    except (ValueError, OSError): pass
     # We may use pipes here (ensuring we use dup to inherit handles), but those have limited buffer
     # so we'll need to poll process
     if stdin:
         with open(stdin_path, "wb+") as stdinf:
-            stdinf.write(stdin)
+            stdinf.write(stdin.encode() if isinstance(stdin, str) else stdin)
         stdin_fl = os.open(stdin_path, os.O_RDONLY | os.O_BINARY)
         stdin_no = sys.stdin.fileno()
         stdin_cp = os.dup(stdin_no)
@@ -120,6 +128,11 @@ def run_proc_windows(proc, params, stdin=None):
     stderr_fl = os.open(stderr_path, os.O_CREAT | os.O_RDWR | os.O_BINARY)
     stderr_no = sys.stderr.fileno()
     stderr_cp = os.dup(stderr_no)
+    if passfo:
+        with open(pass_path, "w+") as passf:
+            passf.write(passfo.read())
+        pass_fl = os.open(pass_path, os.O_RDONLY | os.O_BINARY)
+        pass_cp = os.dup(passfd)
 
     try:
         os.dup2(stdout_fl, stdout_no)
@@ -129,6 +142,9 @@ def run_proc_windows(proc, params, stdin=None):
         if stdin:
             os.dup2(stdin_fl, stdin_no)
             os.close(stdin_fl)
+        if passfo:
+            os.dup2(pass_fl, passfd)
+            os.close(pass_fl)
         retcode = os.spawnv(os.P_WAIT, proc, params)
     finally:
         os.dup2(stdout_cp, stdout_no)
@@ -138,12 +154,18 @@ def run_proc_windows(proc, params, stdin=None):
         if stdin:
             os.dup2(stdin_cp, stdin_no)
             os.close(stdin_cp)
+        if passfo:
+            os.dup2(pass_cp, passfd)
+            os.close(pass_cp)
+            passfo.close()
     out = file_text(stdout_path).replace('\r\n', '\n')
     err = file_text(stderr_path).replace('\r\n', '\n')
     os.unlink(stdout_path)
     os.unlink(stderr_path)
     if stdin: 
         os.unlink(stdin_path)
+    if passfo: 
+        os.unlink(pass_path)
     logging.debug(err.strip())
     logging.debug(out.strip())
     return (retcode, out, err)
@@ -154,7 +176,8 @@ def run_proc(proc, params, stdin=None):
         return run_proc_windows(proc, params, stdin)
 
     logging.debug((proc + ' ' + ' '.join(params)).strip())
-    process = Popen([proc] + params, stdout=PIPE, stderr=PIPE, stdin=PIPE if stdin else None)
+    process = Popen([proc] + params, stdout=PIPE, stderr=PIPE,
+                    stdin=PIPE if stdin else None, close_fds=False, universal_newlines=True)
     output, errout = process.communicate(stdin)
     retcode = process.poll()
     logging.debug(errout.strip())

src/tests/cli_perf.py

@@ -55,31 +55,34 @@ def setup(workdir):
     # Creating working directory and populating it with test files
     RNPDIR = path.join(WORKDIR, '.rnp')
     GPGDIR = path.join(WORKDIR, '.gpg')
-    os.mkdir(RNPDIR, 0700)
-    os.mkdir(GPGDIR, 0700)
+    os.mkdir(RNPDIR, 0o700)
+    os.mkdir(GPGDIR, 0o700)
 
     # Generating key
     pipe = pswd_pipe(PASSWORD)
-    params = ['--homedir', RNPDIR, '--pass-fd', str(pipe), '--userid', 'performance@rnp', '--generate-key']
+    params = ['--homedir', RNPDIR, '--pass-fd', str(pipe), '--userid', 'performance@rnp',
+              '--generate-key']
     # Run key generation
     ret, out, err = run_proc(RNPK, params)
     os.close(pipe)
 
     # Importing keys to GnuPG so it can build trustdb and so on
-    ret, out, err = run_proc(GPG, ['--batch', '--passphrase', '', '--homedir', GPGDIR, '--import', path.join(RNPDIR, 'pubring.gpg'), path.join(RNPDIR, 'secring.gpg')])
+    ret, out, err = run_proc(GPG, ['--batch', '--passphrase', '', '--homedir', GPGDIR,
+                                   '--import', path.join(RNPDIR, 'pubring.gpg'),
+                                   path.join(RNPDIR, 'secring.gpg')])
 
     # Generating small file for tests
-    SMALLSIZE = 3312;
-    st = 'lorem ipsum dol ' * (SMALLSIZE/16)
+    SMALLSIZE = 3312
+    st = 'lorem ipsum dol ' * (SMALLSIZE//16+1)
     with open(path.join(WORKDIR, SMALLFILE), 'w+') as small_file:
         small_file.write(st)
 
     # Generating large file for tests
-    print 'Generating large file of size {}'.format(size_to_readable(LARGESIZE))
+    print('Generating large file of size {}'.format(size_to_readable(LARGESIZE)))
 
-    st = '0123456789ABCDEF' * (1024/16)
+    st = '0123456789ABCDEF' * (1024//16)
     with open(path.join(WORKDIR, LARGEFILE), 'w') as fd:
-        for i in range(0, LARGESIZE / 1024 - 1):
+        for i in range(0, LARGESIZE // 1024):
             fd.write(st)
 
 def run_iterated(iterations, func, src, dst, *args):
@@ -96,28 +99,34 @@ def run_iterated(iterations, func, src, dst, *args):
     return res
 
 def rnp_symencrypt_file(src, dst, cipher, zlevel = 6, zalgo = 'zip', armor = False):
-    params = ['--homedir', RNPDIR, '--password', PASSWORD, '--cipher', cipher, '-z', str(zlevel), '--' + zalgo, '-c', src, '--output', dst]
+    params = ['--homedir', RNPDIR, '--password', PASSWORD, '--cipher', cipher,
+              '-z', str(zlevel), '--' + zalgo, '-c', src, '--output', dst]
     if armor:
         params += ['--armor']
     ret = run_proc_fast(RNP, params)
     if ret != 0:
         raise_err('rnp symmetric encryption failed')
 
 def rnp_decrypt_file(src, dst):
-    ret = run_proc_fast(RNP, ['--homedir', RNPDIR, '--password', PASSWORD, '--decrypt', src, '--output', dst])
+    ret = run_proc_fast(RNP, ['--homedir', RNPDIR, '--password', PASSWORD, '--decrypt', src,
+                              '--output', dst])
     if ret != 0:
         raise_err('rnp decryption failed')
 
 def gpg_symencrypt_file(src, dst, cipher = 'AES', zlevel = 6, zalgo = 1, armor = False):
-    params = ['--homedir', GPGDIR, '-c', '-z', str(zlevel), '--s2k-count', '524288', '--compress-algo', str(zalgo), '--batch', '--passphrase', PASSWORD, '--cipher-algo', cipher, '--output', dst, src]
+    params = ['--homedir', GPGDIR, '-c', '-z', str(zlevel), '--s2k-count', '524288',
+              '--compress-algo', str(zalgo), '--batch', '--passphrase', PASSWORD,
+              '--cipher-algo', cipher, '--output', dst, src]
     if armor:
         params.insert(2, '--armor')
     ret = run_proc_fast(GPG, params)
     if ret != 0:
         raise_err('gpg symmetric encryption failed for cipher ' + cipher)
 
 def gpg_decrypt_file(src, dst, keypass):
-    ret = run_proc_fast(GPG, ['--homedir', GPGDIR, '--pinentry-mode=loopback', '--batch', '--yes', '--passphrase', keypass, '--trust-model', 'always', '-o', dst, '-d', src])
+    ret = run_proc_fast(GPG, ['--homedir', GPGDIR, '--pinentry-mode=loopback', '--batch',
+                              '--yes', '--passphrase', keypass, '--trust-model', 'always',
+                              '-o', dst, '-d', src])
     if ret != 0:
         raise_err('gpg decryption failed')
 
@@ -132,27 +141,33 @@ def print_test_results(fsize, rnptime, gpgtime, operation):
 
         if rnpruns >= gpgruns:
             percents = (rnpruns - gpgruns) / gpgruns * 100
-            logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(operation, percents, runstr))
+            logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(
+                operation, percents, runstr))
         else:
             percents = (gpgruns - rnpruns) / gpgruns * 100
-            logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(operation, percents, runstr))
+            logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(
+                operation, percents, runstr))
     else:
         rnpspeed = fsize / 1024.0 / 1024.0 / rnptime
         gpgspeed = fsize / 1024.0 / 1024.0 / gpgtime
         spdstr = '{:.2f} MB/sec vs {:.2f} MB/sec'.format(rnpspeed, gpgspeed)
 
         if rnpspeed >= gpgspeed:
             percents = (rnpspeed - gpgspeed) / gpgspeed * 100
-            logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(operation, percents, spdstr))
+            logging.info('{:<30}: RNP is {:>3.0f}% FASTER then GnuPG ({})'.format(
+                operation, percents, spdstr))
         else:
             percents = (gpgspeed - rnpspeed) / gpgspeed * 100
-            logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(operation, percents, spdstr))
+            logging.info('{:<30}: RNP is {:>3.0f}% SLOWER then GnuPG ({})'.format(
+                operation, percents, spdstr))
 
 def get_file_params(filetype):
     if filetype == 'small':
-        infile, outfile, iterations, fsize = (SMALLFILE, SMALLFILE + '.gpg', SMALL_ITERATIONS, SMALLSIZE)
+        infile, outfile, iterations, fsize = (SMALLFILE, SMALLFILE + '.gpg',
+                                              SMALL_ITERATIONS, SMALLSIZE)
     else:
-        infile, outfile, iterations, fsize = (LARGEFILE, LARGEFILE + '.gpg', LARGE_ITERATIONS, LARGESIZE)
+        infile, outfile, iterations, fsize = (LARGEFILE, LARGEFILE + '.gpg',
+                                              LARGE_ITERATIONS, LARGESIZE)
 
     infile = path.join(WORKDIR, infile)
     rnpout = path.join(WORKDIR, outfile + '.rnp')
@@ -172,8 +187,10 @@ def small_file_symmetric_encryption(self):
         '''
         infile, rnpout, gpgout, iterations, fsize = get_file_params('small')
         for armor in [False, True]:
-            tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout, 'AES128', 0, 'zip', armor)
-            tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout, 'AES128', 0, 1, armor)
+            tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout,
+                                 'AES128', 0, 'zip', armor)
+            tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout,
+                                 'AES128', 0, 1, armor)
             testname = 'ENCRYPT-SMALL-{}'.format('ARMOR' if armor else 'BINARY')
             print_test_results(fsize, tmrnp, tmgpg, testname)
 
@@ -183,8 +200,10 @@ def large_file_symmetric_encryption(self):
         '''
         infile, rnpout, gpgout, iterations, fsize = get_file_params('large')
         for cipher in ['AES128', 'AES192', 'AES256', 'TWOFISH', 'BLOWFISH', 'CAST5', 'CAMELLIA128', 'CAMELLIA192', 'CAMELLIA256']:
-            tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout, cipher, 0, 'zip', False)
-            tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout, cipher, 0, 1, False)
+            tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout,
+                                 cipher, 0, 'zip', False)
+            tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout,
+                                 cipher, 0, 1, False)
             testname = 'ENCRYPT-{}-BINARY'.format(cipher)
             print_test_results(fsize, tmrnp, tmgpg, testname)
 
@@ -193,7 +212,8 @@ def large_file_armored_encryption(self):
         Large file armored encryption
         '''
         infile, rnpout, gpgout, iterations, fsize = get_file_params('large')
-        tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout, 'AES128', 0, 'zip', True)
+        tmrnp = run_iterated(iterations, rnp_symencrypt_file, infile, rnpout,
+                             'AES128', 0, 'zip', True)
         tmgpg = run_iterated(iterations, gpg_symencrypt_file, infile, gpgout, 'AES128', 0, 1, True)
         print_test_results(fsize, tmrnp, tmgpg, 'ENCRYPT-LARGE-ARMOR')
 
@@ -217,7 +237,8 @@ def large_file_symmetric_decryption(self):
         '''
         infile, rnpout, gpgout, iterations, fsize = get_file_params('large')
         inenc = infile + '.enc'
-        for cipher in ['AES128', 'AES192', 'AES256', 'TWOFISH', 'BLOWFISH', 'CAST5', 'CAMELLIA128', 'CAMELLIA192', 'CAMELLIA256']:
+        for cipher in ['AES128', 'AES192', 'AES256', 'TWOFISH', 'BLOWFISH', 'CAST5',
+                       'CAMELLIA128', 'CAMELLIA192', 'CAMELLIA256']:
             gpg_symencrypt_file(infile, inenc, cipher, 0, 1, False)
             tmrnp = run_iterated(iterations, rnp_decrypt_file, inenc, rnpout)
             tmgpg = run_iterated(iterations, gpg_decrypt_file, inenc, gpgout, PASSWORD)
@@ -266,11 +287,14 @@ def large_file_armored_decryption(self):
                       help="Name of the comma-separated benchmarks to run", metavar="benchmarks")
     parser.add_argument("-w", "--workdir", dest="workdir",
                       help="Working directory to use", metavar="workdir")
-    parser.add_argument("-l", "--list", help="Print list of available benchmarks and exit", action="store_true")
+    parser.add_argument("-l", "--list", help="Print list of available benchmarks and exit",
+                        action="store_true")
     args = parser.parse_args()
 
     # get list of benchamrks to run
-    bench_methods = [ x[0] for x in inspect.getmembers(Benchmark,predicate=inspect.ismethod) if not x[0].startswith("_") ]
+    bench_methods = [ x[0] for x in inspect.getmembers(Benchmark,
+            predicate=lambda x: inspect.ismethod(x) or inspect.isfunction(x))]
+    print(bench_methods)
 
     if args.list:
         for name in bench_methods: