cleanup and enable py39 tests

Author: wdpypere

bin/zkinitree.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: latin-1 -*-
 #
 # Copyright 2013-2023 Ghent University
 #
@@ -64,9 +63,9 @@ def main():
     # Iterate paths
     for path, attrs in znodes.iteritems():
         logger.debug("path %s attribs %s", path, attrs)
-        acls = dict((arg, attrs[arg]) for arg in attrs if arg not in ('value', 'ephemeral', 'sequence', 'makepath'))
+        acls = {arg: attrs[arg] for arg in attrs if arg not in ('value', 'ephemeral', 'sequence', 'makepath')}
         acl_list = parse_acls(acls, users, root_acl)
-        kwargs = dict((arg, attrs[arg]) for arg in attrs if arg in ('ephemeral', 'sequence', 'makepath'))
+        kwargs = {arg: attrs[arg] for arg in attrs if arg in ('ephemeral', 'sequence', 'makepath')}
         if not zkclient.exists_znode(path):
             zkclient.make_znode(path, value=attrs.get('value', ''), acl=acl_list, **kwargs)
         else:

bin/zkrsync.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: latin-1 -*-
 #
 # Copyright 2013-2023 Ghent University
 #
@@ -239,7 +238,7 @@ def main():
         # Pathbuilding (Source clients and pathsonly ) specific options:
         'rsubpaths'   : ('rsync subpaths, specified as <depth>_<path>, with deepest paths last',
                             'strlist', 'store', None),
-        'excludere'   : ('Exclude from pathbuilding', None, 'regex', re.compile('/\.snapshots(/.*|$)')),
+        'excludere'   : ('Exclude from pathbuilding', None, 'regex', re.compile(r'/\.snapshots(/.*|$)')),
         'excl_usr'    : ('If set, exclude paths for this user only when using excludere', None, 'store', 'root'),
         'depth'       : ('queue depth', "int", 'store', 3),
         # Source clients options; should be the same on all clients of the session!:

lib/vsc/zk/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: latin-1 -*-
 #
 # Copyright 2009-2023 Ghent University
 #

lib/vsc/zk/base.py

@@ -1,4 +1,3 @@
-# -*- coding: latin-1 -*-
 #
 # Copyright 2013-2023 Ghent University
 #
@@ -48,7 +47,7 @@ class RunWatchLoopLog(RunAsyncLoopLog):
     def __init__(self, cmd, **kwargs):
 
         self.watchclient = kwargs.pop('watchclient', None)
-        super(RunWatchLoopLog, self).__init__(cmd, **kwargs)
+        super().__init__(cmd, **kwargs)
         self.log.debug("watchclient %s registered", self.watchclient)
 
 
@@ -57,7 +56,7 @@ def _loop_process_output(self, output):
         send it to the logger. The logger need to be stream-like
         When watch is ready, stop
         """
-        super(RunWatchLoopLog, self)._loop_process_output(output)
+        super()._loop_process_output(output)
         # self.log.debug("watchclient status %s" % self.watchclient.is_ready())
         if self.watchclient.is_ready():
             self.log.debug("watchclient %s ready", self.watchclient)
@@ -87,7 +86,7 @@ def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=N
             'auth_data'   : auth_data,
           #  'logger'      : self.log
         }
-        super(VscKazooClient, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.start()
         self.log.debug('Zookeeper client started')
 
@@ -117,7 +116,7 @@ def join_parties(self, parties=None):
             self.log.debug("Joining %s parties: %s", len(parties), ", ".join(parties))
             for party in parties:
 
-                partypath = '%s/%s/parties/%s' % (self.BASE_ZNODE, self.session, party)
+                partypath = f'{self.BASE_ZNODE}/{self.session}/parties/{party}'
                 thisparty = Party(self, partypath, self.whoami)
                 thisparty.join()
                 self.parties[party] = thisparty
@@ -133,11 +132,11 @@ def znode_path(self, znode=None):
         if isinstance(znode, str):
             if not znode.startswith(base_znode_string):
                 if not znode.startswith('/'):
-                    znode = '%s/%s' % (self.BASE_ZNODE, znode)
+                    znode = f'{self.BASE_ZNODE}/{znode}'
                 else:
-                    self.log.raiseException('path %s not subpath of %s ' % (znode, base_znode_string))
+                    self.log.raiseException(f'path {znode} not subpath of {base_znode_string} ')
         else:
-            self.log.raiseException('Unsupported znode type %s (znode %s)' % (znode, type(znode)))
+            self.log.raiseException(f'Unsupported znode type {znode} (znode {type(znode)})')
         self.log.debug("znode %s", znode)
         return znode
 
@@ -148,9 +147,9 @@ def make_znode(self, znode=None, value="", acl=None, **kwargs):
         try:
             znode = self.create(znode_path, value=value.encode(), acl=acl, **kwargs)
         except NodeExistsError:
-            self.log.raiseException('znode %s already exists' % znode_path)
+            self.log.raiseException(f'znode {znode_path} already exists')
         except NoNodeError:
-            self.log.raiseException('parent node(s) of znode %s missing' % znode_path)
+            self.log.raiseException(f'parent node(s) of znode {znode_path} missing')
 
         self.log.debug("znode %s created in zookeeper", znode)
         return znode
@@ -177,28 +176,28 @@ def znode_acls(self, znode=None, acl=None):
         try:
             self.set_acls(znode_path, acl)
         except NoAuthError:
-            self.log.raiseException('No valid authentication for (%s) on path %s!' % (self.auth_data, znode_path))
+            self.log.raiseException(f'No valid authentication for ({self.auth_data}) on path {znode_path}!')
         except NoNodeError:
-            self.log.raiseException('node %s doesn\'t exists' % znode_path)
+            self.log.raiseException(f'node {znode_path} doesn\'t exists')
 
         self.log.debug("added ACL for znode %s in zookeeper", znode_path)
 
     def set_watch_value(self, watch, value):
         """Sets the value of an existing watch"""
-        watchpath = '%s/%s' % (self.watchpath, watch)
+        watchpath = f'{self.watchpath}/{watch}'
         self.set(watchpath, value.encode())
 
     def new_watch(self, watch, value):
         """ Start a watch other clients can register to """
-        watchpath = '%s/%s' % (self.watchpath, watch)
+        watchpath = f'{self.watchpath}/{watch}'
         if self.exists(watchpath):
             self.log.error('watchnode %s already exists!', watchpath)
             return False
         return self.make_znode(watchpath, value, makepath=True)
 
     def remove_watch(self, watch):
         """ Removes a wath """
-        watchpath = '%s/%s' % (self.watchpath, watch)
+        watchpath = f'{self.watchpath}/{watch}'
         self.delete(watchpath)
 
     def set_ready(self):
@@ -228,7 +227,7 @@ def exit(self):
 
     def ready_with_stop_watch(self):
         """Register to watch and set to ready when watch is set to 'stop' """
-        watchpath = '%s/%s' % (self.watchpath, 'ready')
+        watchpath = f'{self.watchpath}/ready'
         @self.DataWatch(watchpath)
         # pylint: disable=unused-variable,unused-argument
         def ready_watcher(data, stat):

lib/vsc/zk/depthwalk.py

@@ -1,4 +1,3 @@
-# -*- coding: latin-1 -*-
 #
 # Copyright 2013-2023 Ghent University
 #
@@ -125,9 +124,9 @@ def get_pathlist(path, depth, exclude_re=None, exclude_usr=None, rsubpaths=None)
             subdepth, subpath = encsubpath.split('_', 1)
             subpath = subpath.rstrip(os.path.sep)
             if not subpath.startswith(path):
-                subpath = '%s/%s' % (path, subpath)
+                subpath = f'{path}/{subpath}'
             if subpath not in pathdict:
-                logger.raiseException('%s is not in the pathlist of %s with depth %d!' % (subpath, path, depth))
+                logger.raiseException(f'{subpath} is not in the pathlist of {path} with depth {int(depth)}!')
 
             subpathdepth = subpath.count(os.path.sep)
             newdepth = subpathdepth + int(subdepth)
@@ -147,7 +146,7 @@ def get_pathlist(path, depth, exclude_re=None, exclude_usr=None, rsubpaths=None)
 def encode_paths(pathlist):
     enclist = []
     for (path, rec) in pathlist:
-        enclist.append("%i_%s" % (rec, path))
+        enclist.append(f"{int(rec)}_{path}")
     logger.debug("encoded list is %s", enclist)
     return enclist
 

lib/vsc/zk/rsync/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: latin-1 -*-
 #
 # Copyright 2009-2023 Ghent University
 #

lib/vsc/zk/rsync/controller.py

@@ -1,4 +1,3 @@
-# -*- coding: latin-1 -*-
 #
 # Copyright 2013-2023 Ghent University
 #
@@ -63,16 +62,16 @@ def __init__(self, hosts, session=None, name=None, default_acl=None,
         }
         self.netcat = netcat
 
-        super(RsyncController, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.rsyncpath = rsyncpath.rstrip(os.path.sep)
 
         if not netcat:
             if verifypath and not self.basepath_ok():
-                self.log.raiseException('Path does not exists in filesystem: %s' % rsyncpath)
+                self.log.raiseException(f'Path does not exists in filesystem: {rsyncpath}')
             if not os.path.isdir(self.RSDIR):
                 os.mkdir(self.RSDIR, 0o700)
-            self.module = 'zkrs-%s' % self.session
+            self.module = f'zkrs-{self.session}'
 
         self.dest_queue = LockingQueue(self, self.znode_path(self.session + '/destQueue'))
         self.verifypath = verifypath
@@ -90,10 +89,10 @@ def basepath_ok(self):
 
     def dest_state(self, dest, state):
         """ Set the destination to a different state """
-        destdir = '%s/dests' % self.session
+        destdir = f'{self.session}/dests'
         self.ensure_path(self.znode_path(destdir))
         lock = self.Lock(self.znode_path(self.session + '/destslock'), dest)
-        destpath = '%s/%s' % (destdir, dest)
+        destpath = f'{destdir}/{dest}'
         with lock:
             if not self.exists_znode(destpath):
                 self.make_znode(destpath, ephemeral=True)
@@ -110,4 +109,3 @@ def dest_state(self, dest, state):
             else:
                 self.log.error('No valid state: %s ', state)
                 return None
-

lib/vsc/zk/rsync/destination.py

@@ -1,4 +1,3 @@
-# -*- coding: latin-1 -*-
 #
 # Copyright 2013-2023 Ghent University
 #
@@ -46,7 +45,7 @@ class RunDestination(RunWatchLoopLog):
 
     def __init__(self, cmd, **kwargs):
 
-        super(RunDestination, self).__init__(cmd, **kwargs)
+        super().__init__(cmd, **kwargs)
         self.registered = False
         self.paused = False
 
@@ -56,7 +55,7 @@ def _loop_process_output(self, output):
         Register destination after 2 loops.
         When watch is ready, stop
         """
-        super(RunDestination, self)._loop_process_output(output)
+        super()._loop_process_output(output)
 
         if self.watchclient.verifypath and (self._loop_count % self.WAITLOOPS == 0):
             if self.watchclient.basepath_ok():
@@ -102,13 +101,13 @@ def __init__(self, hosts, session=None, name=None, default_acl=None,
         host = socket.getfqdn()
         if domain:
             hname = host.split('.', 1)
-            host = '%s.%s' % (hname[0], domain)
+            host = f'{hname[0]}.{domain}'
         self.daemon_host = host
         self.daemon_port = rsyncport
         self.start_port = startport
         self.port = None
 
-        super(RsyncDestination, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
     def get_whoami(self, name=None):  # Override base method
         """Create a unique name for this client"""
@@ -141,25 +140,25 @@ def generate_daemon_config(self):
 
     def reserve_port(self):
         """ Search for an available port """
-        portdir = '%s/usedports/%s' % (self.session, self.daemon_host)
+        portdir = f'{self.session}/usedports/{self.daemon_host}'
         self.ensure_path(self.znode_path(portdir))
         lock = self.Lock(self.znode_path(self.session + '/portlock'), self.whoami)
         with lock:
             if self.daemon_port:
                 port = self.daemon_port
-                portpath = '%s/%s' % (portdir, port)
+                portpath = f'{portdir}/{port}'
                 if self.exists_znode(portpath):
                     self.log.raiseException('Port already in use: %s', port)
 
             else:
                 port = self.start_port
-                portpath = '%s/%s' % (portdir, port)
-                while (self.exists_znode(portpath)):
+                portpath = f'{portdir}/{port}'
+                while self.exists_znode(portpath):
                     port += 1
-                    portpath = '%s/%s' % (portdir, port)
+                    portpath = f'{portdir}/{port}'
 
             self.make_znode(portpath, ephemeral=True)
-            portmap = '%s/portmap/%s' % (self.session, self.whoami)
+            portmap = f'{self.session}/portmap/{self.whoami}'
             if not self.exists_znode(portmap):
                 self.make_znode(portmap, ephemeral=True, makepath=True)
             self.set_znode(portmap, str(port))
@@ -228,7 +227,7 @@ def run(self, attempts=3):
 
     def add_to_queue(self):
         """Add this destination to the destination queue """
-        destid = '%d:%s' % (self.port, self.whoami)
+        destid = f'{int(self.port)}:{self.whoami}'
         self.dest_queue.put(destid.encode())
         self.log.info('Added destination %s to queue with port %s', self.whoami, self.port)