run pyupgrade --py38-plus

Author: branchvincent

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # python-scrapinghub documentation build configuration file, created by
 # sphinx-quickstart on Fri Mar 24 12:28:40 2017.
@@ -58,9 +57,9 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'scrapinghub'
-copyright = u'2010-{}, Scrapinghub'.format(YEAR)
-author = u'Scrapinghub'
+project = 'scrapinghub'
+copyright = f'2010-{YEAR}, Scrapinghub'
+author = 'Scrapinghub'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -140,8 +139,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'python-scrapinghub.tex', u'python-scrapinghub Documentation',
-     u'Pablo Hoffman, Daniel Graña', 'manual'),
+    (master_doc, 'python-scrapinghub.tex', 'python-scrapinghub Documentation',
+     'Pablo Hoffman, Daniel Graña', 'manual'),
 ]
 
 
@@ -150,7 +149,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'python-scrapinghub', u'python-scrapinghub Documentation',
+    (master_doc, 'python-scrapinghub', 'python-scrapinghub Documentation',
      [author], 1)
 ]
 
@@ -161,7 +160,7 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'python-scrapinghub', u'python-scrapinghub Documentation',
+    (master_doc, 'python-scrapinghub', 'python-scrapinghub Documentation',
      author, 'python-scrapinghub', 'One line description of project.',
      'Miscellaneous'),
 ]
@@ -189,7 +188,7 @@ def handle_item(fieldarg, content):
             # inconsistencies later when references are resolved
             fieldtype = types.pop(fieldarg)
             if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text):
-                typename = u''.join(n.astext() for n in fieldtype)
+                typename = ''.join(n.astext() for n in fieldtype)
                 par.extend(self.make_xrefs(self.typerolename, domain, typename,
                                            addnodes.literal_emphasis))
             else:

scrapinghub/client/__init__.py

@@ -16,18 +16,18 @@ class Connection(_Connection):
 
     @_wrap_http_errors
     def _request(self, *args, **kwargs):
-        return super(Connection, self)._request(*args, **kwargs)
+        return super()._request(*args, **kwargs)
 
 
 class HubstorageClient(_HubstorageClient):
 
     @_wrap_http_errors
     def request(self, *args, **kwargs):
-        return super(HubstorageClient, self).request(*args, **kwargs)
+        return super().request(*args, **kwargs)
 
 
-class ScrapinghubClient(object):
-    """Main class to work with Scrapinghub API.
+class ScrapinghubClient:
+    r"""Main class to work with Scrapinghub API.
 
     :param auth: (optional) Scrapinghub APIKEY or other SH auth credentials.
         If not provided, it will read, respectively, from 

scrapinghub/client/activity.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from .proxy import _Proxy
 from .utils import parse_job_key, update_kwargs
 

scrapinghub/client/collections.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from six import string_types
 from six.moves import collections_abc
 
@@ -96,7 +94,7 @@ def list(self):
         return list(self.iter())
 
 
-class Collection(object):
+class Collection:
     """Representation of a project collection object.
 
     Not a public constructor: use :class:`Collections` instance to get a
@@ -157,7 +155,7 @@ def __init__(self, client, collections, type_, name):
         self._origin = _Collection(type_, name, collections._origin)
 
     def get(self, key, **params):
-        """Get item from collection by key.
+        r"""Get item from collection by key.
 
         :param key: string item key.
         :param \*\*params: (optional) additional query params for the request.
@@ -184,7 +182,7 @@ def delete(self, keys):
 
         The method returns ``None`` (original method returns an empty generator).
         """
-        if (not isinstance(keys, string_types) and
+        if (not isinstance(keys, str) and
                 not isinstance(keys, collections_abc.Iterable)):
             raise ValueError("You should provide string key or iterable "
                              "object providing string keys")
@@ -209,7 +207,7 @@ def count(self, *args, **kwargs):
 
     def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
              endts=None, requests_params=None, **params):
-        """A method to iterate through collection items.
+        r"""A method to iterate through collection items.
 
         :param key: a string key or a list of keys to filter with.
         :param prefix: a string prefix to filter items.
@@ -230,7 +228,7 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
 
     def list(self, key=None, prefix=None, prefixcount=None, startts=None,
              endts=None, requests_params=None, **params):
-        """Convenient shortcut to list iter results.
+        r"""Convenient shortcut to list iter results.
 
         Please note that :meth:`list` method can use a lot of memory and for a
         large amount of logs it's recommended to iterate through it

scrapinghub/client/exceptions.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import
 from functools import wraps
 
 from requests import HTTPError
@@ -29,7 +27,7 @@ def __init__(self, message=None, http_error=None):
         self.http_error = http_error
         if not message:
             message = _get_http_error_msg(http_error)
-        super(ScrapinghubAPIError, self).__init__(message)
+        super().__init__(message)
 
 
 class BadRequest(ScrapinghubAPIError):

scrapinghub/client/frontiers.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from functools import partial
 from collections import defaultdict
 
@@ -15,7 +14,7 @@ class _HSFrontier(_Frontier):
     """Modified hubstorage Frontier with newcount per slot."""
 
     def __init__(self, *args, **kwargs):
-        super(_HSFrontier, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.newcount = defaultdict(int)
 
     def _get_writer(self, frontier, slot):
@@ -84,7 +83,7 @@ class Frontiers(_Proxy):
         >>> project.frontiers.close()
     """
     def __init__(self, *args, **kwargs):
-        super(Frontiers, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def get(self, name):
         """Get a frontier by name.
@@ -125,7 +124,7 @@ def close(self):
         self._origin.close()
 
 
-class Frontier(object):
+class Frontier:
     """Representation of a frontier object.
 
     Not a public constructor: use :class:`Frontiers` instance to get a
@@ -201,7 +200,7 @@ def newcount(self):
                    if frontier == self.key)
 
 
-class FrontierSlot(object):
+class FrontierSlot:
     """Representation of a frontier slot object.
 
     Not a public constructor: use :class:`Frontier` instance to get a
@@ -295,7 +294,7 @@ def newcount(self):
         return newcount_values.get((self._frontier.key, self.key), 0)
 
 
-class FrontierSlotFingerprints(object):
+class FrontierSlotFingerprints:
     """Representation of request fingerprints collection stored in slot."""
 
     def __init__(self, slot):
@@ -311,13 +310,13 @@ def add(self, fps):
         origin = self._frontier._frontiers._origin
         writer = origin._get_writer(self._frontier.key, self.key)
         fps = list(fps) if not isinstance(fps, list) else fps
-        if not all(isinstance(fp, string_types) for fp in fps):
+        if not all(isinstance(fp, str) for fp in fps):
             raise ValueError('Fingerprint should be of a string type')
         for fp in fps:
             writer.write({'fp': fp})
 
     def iter(self, **params):
-        """Iterate through fingerprints in the slot.
+        r"""Iterate through fingerprints in the slot.
 
         :param \*\*params: (optional) additional query params for the request.
         :return: an iterator over fingerprints.
@@ -329,7 +328,7 @@ def iter(self, **params):
             yield fp.get('fp')
 
     def list(self, **params):
-        """List fingerprints in the slot.
+        r"""List fingerprints in the slot.
 
         :param \*\*params: (optional) additional query params for the request.
         :return: a list of fingerprints.
@@ -338,7 +337,7 @@ def list(self, **params):
         return list(self.iter(**params))
 
 
-class FrontierSlotQueue(object):
+class FrontierSlotQueue:
     """Representation of request batches queue stored in slot."""
 
     def __init__(self, slot):
@@ -352,7 +351,7 @@ def add(self, fps):
         return origin.add(self._frontier.key, self.key, fps)
 
     def iter(self, mincount=None, **params):
-        """Iterate through batches in the queue.
+        r"""Iterate through batches in the queue.
 
         :param mincount: (optional) limit results with min amount of requests.
         :param \*\*params: (optional) additional query params for the request.
@@ -366,7 +365,7 @@ def iter(self, mincount=None, **params):
         return origin.apiget(path, params=params)
 
     def list(self, mincount=None, **params):
-        """List request batches in the queue.
+        r"""List request batches in the queue.
 
         :param mincount: (optional) limit results with min amount of requests.
         :param \*\*params: (optional) additional query params for the request.

scrapinghub/client/items.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import sys
 
 from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin
@@ -84,10 +82,10 @@ def _modify_iter_params(self, params):
         :return: a dict with updated set of params.
         :rtype: :class:`dict`
         """
-        params = super(Items, self)._modify_iter_params(params)
+        params = super()._modify_iter_params(params)
         offset = params.pop('offset', None)
         if offset:
-            params['start'] = '{}/{}'.format(self.key, offset)
+            params['start'] = f'{self.key}/{offset}'
         return params
 
     def list_iter(self, chunksize=1000, *args, **kwargs):

scrapinghub/client/jobs.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import json
 
 from ..hubstorage.job import JobMeta as _JobMeta
@@ -17,7 +15,7 @@
 from .utils import get_tags_for_update, parse_job_key, update_kwargs
 
 
-class Jobs(object):
+class Jobs:
     """Class representing a collection of jobs for a project/spider.
 
     Not a public constructor: use :class:`~scrapinghub.client.projects.Project`
@@ -45,7 +43,7 @@ def __init__(self, client, project_id, spider=None):
 
     def count(self, spider=None, state=None, has_tag=None, lacks_tag=None,
               startts=None, endts=None, **params):
-        """Count jobs with a given set of filters.
+        r"""Count jobs with a given set of filters.
 
         :param spider: (optional) filter by spider name.
         :param state: (optional) a job state, a string or a list of strings.
@@ -140,7 +138,7 @@ def cancel(self, keys=None, count=None, **params):
     def iter(self, count=None, start=None, spider=None, state=None,
              has_tag=None, lacks_tag=None, startts=None, endts=None,
              meta=None, **params):
-        """Iterate over jobs collection for a given set of params.
+        r"""Iterate over jobs collection for a given set of params.
 
         :param count: (optional) limit amount of returned jobs.
         :param start: (optional) number of jobs to skip in the beginning.
@@ -211,7 +209,7 @@ def iter(self, count=None, start=None, spider=None, state=None,
     def list(self, count=None, start=None, spider=None, state=None,
              has_tag=None, lacks_tag=None, startts=None, endts=None,
              meta=None, **params):
-        """Convenient shortcut to list iter results.
+        r"""Convenient shortcut to list iter results.
 
         :param count: (optional) limit amount of returned jobs.
         :param start: (optional) number of jobs to skip in the beginning.
@@ -250,7 +248,7 @@ def list(self, count=None, start=None, spider=None, state=None,
     def run(self, spider=None, units=None, priority=None, meta=None,
             add_tag=None, job_args=None, job_settings=None, cmd_args=None,
             environment=None, **params):
-        """Schedule a new job and returns its job key.
+        r"""Schedule a new job and returns its job key.
 
         :param spider: a spider name string
             (not needed if job is scheduled via :attr:`Spider.jobs`).
@@ -329,7 +327,7 @@ def get(self, job_key):
         return Job(self._client, str(job_key))
 
     def summary(self, state=None, spider=None, **params):
-        """Get jobs summary (optionally by state).
+        r"""Get jobs summary (optionally by state).
 
         :param state: (optional) a string state to filter jobs.
         :param spider: (optional) a spider name (not needed if instantiated
@@ -355,7 +353,7 @@ def summary(self, state=None, spider=None, **params):
 
     def iter_last(self, start=None, start_after=None, count=None,
                   spider=None, **params):
-        """Iterate through last jobs for each spider.
+        r"""Iterate through last jobs for each spider.
 
         :param start: (optional)
         :param start_after: (optional)
@@ -445,7 +443,7 @@ def update_tags(self, add=None, remove=None, spider=None):
         return result['count']
 
 
-class Job(object):
+class Job:
     """Class representing a job object.
 
     Not a public constructor: use :class:`~scrapinghub.client.ScrapinghubClient`
@@ -510,7 +508,7 @@ def close_writers(self):
         self._job.close_writers()
 
     def start(self, **params):
-        """Move job to running state.
+        r"""Move job to running state.
 
         :param \*\*params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
@@ -524,7 +522,7 @@ def start(self, **params):
         return self.update(state='running', **params)
 
     def finish(self, **params):
-        """Move running job to finished state.
+        r"""Move running job to finished state.
 
         :param \*\*params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
@@ -538,7 +536,7 @@ def finish(self, **params):
         return self.update(state='finished', **params)
 
     def delete(self, **params):
-        """Mark finished job for deletion.
+        r"""Mark finished job for deletion.
 
         :param \*\*params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
@@ -552,7 +550,7 @@ def delete(self, **params):
         return self.update(state='deleted', **params)
 
     def update(self, state, **params):
-        """Update job state.
+        r"""Update job state.
 
         :param state: a new job state.
         :param \*\*params: (optional) keyword meta parameters to update.
@@ -568,7 +566,7 @@ def update(self, state, **params):
             job = next(self._project.jobq.update(self, state=state, **params))
             return job['prevstate']
         except StopIteration:
-            raise NotFound("Job {} doesn't exist".format(self.key))
+            raise NotFound(f"Job {self.key} doesn't exist")
 
     def cancel(self):
         """Schedule a running job for cancellation.