add automatic caching for discovery requests, refreshing on a miss (#238)

* add automatic caching for discovery requests, refreshing on a miss

* fix base64 encode in python3

* Don't replace ResourceContainer on cache invalidation so that the second attempt succeeds

* Use more generic temp directory and path operations

(cherry picked from commit c6f2168)

Author: fabianvf

openshift/dynamic/client.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 
+import os
 import sys
 import copy
 import json
+import base64
+import tempfile
 from functools import partial
-from six import PY2
+from six import PY2, PY3
 
 import yaml
 from pprint import pformat
@@ -32,6 +35,28 @@
     'ResourceField',
 ]
 
+class CacheEncoder(json.JSONEncoder):
+
+    def default(self, o):
+        return o.to_dict()
+
+def cache_decoder(client):
+
+    class CacheDecoder(json.JSONDecoder):
+        def __init__(self, *args, **kwargs):
+            json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
+
+        def object_hook(self, obj):
+            if '_type' not in obj:
+                return obj
+            _type = obj.pop('_type')
+            if _type == 'Resource':
+                return Resource(client=client, **obj)
+            elif _type == 'ResourceList':
+                return ResourceList(obj['resource'])
+            return obj
+
+    return CacheDecoder
 
 def meta_request(func):
     """ Handles parsing response structure and translating API Exceptions """
@@ -66,18 +91,45 @@ class DynamicClient(object):
         the kubernetes API
     """
 
-    def __init__(self, client):
+    def __init__(self, client, cache_file=None):
         self.client = client
         self.configuration = client.configuration
+        default_cache_id = self.configuration.host
+        if PY3:
+            default_cache_id = default_cache_id.encode('utf-8')
+        default_cachefile_name = 'osrcp-{0}.json'.format(base64.b64encode(default_cache_id).decode('utf-8'))
+        self.__resources = ResourceContainer({}, client=self)
+        self.__cache_file = cache_file or os.path.join(tempfile.gettempdir(), default_cachefile_name)
+        self.__init_cache()
+
+    def __init_cache(self, refresh=False):
+        if refresh or not os.path.exists(self.__cache_file):
+            self.__cache = {}
+            refresh = True
+        else:
+            with open(self.__cache_file, 'r') as f:
+                self.__cache = json.load(f, cls=cache_decoder(self))
         self._load_server_info()
-        self.__resources = ResourceContainer(self.parse_api_groups())
+        self.__resources.update(self.parse_api_groups())
+
+        if refresh:
+            self.__write_cache()
+
+    def __write_cache(self):
+        with open(self.__cache_file, 'w') as f:
+            json.dump(self.__cache, f, cls=CacheEncoder)
+
+    def invalidate_cache(self):
+        self.__init_cache(refresh=True)
 
     def _load_server_info(self):
-        self.__version = {'kubernetes': load_json(self.request('get', '/version'))}
-        try:
-            self.__version['openshift'] = load_json(self.request('get', '/version/openshift'))
-        except ApiException:
-            pass
+        if not self.__cache.get('version'):
+            self.__cache['version'] = {'kubernetes': load_json(self.request('get', '/version'))}
+            try:
+                self.__cache['version']['openshift'] = load_json(self.request('get', '/version/openshift'))
+            except ApiException:
+                pass
+        self.__version = self.__cache['version']
 
     @property
     def resources(self):
@@ -102,20 +154,22 @@ def default_groups(self):
 
     def parse_api_groups(self):
         """ Discovers all API groups present in the cluster """
-        prefix = 'apis'
-        groups_response = load_json(self.request('GET', '/{}'.format(prefix)))['groups']
-
-        groups = self.default_groups()
-        groups[prefix] = {}
-
-        for group in groups_response:
-            new_group = {}
-            for version_raw in group['versions']:
-                version = version_raw['version']
-                preferred = version_raw == group['preferredVersion']
-                new_group[version] = self.get_resources_for_api_version(prefix, group['name'], version, preferred)
-            groups[prefix][group['name']] = new_group
-        return groups
+        if not self.__cache.get('resources'):
+            prefix = 'apis'
+            groups_response = load_json(self.request('GET', '/{}'.format(prefix)))['groups']
+
+            groups = self.default_groups()
+            groups[prefix] = {}
+
+            for group in groups_response:
+                new_group = {}
+                for version_raw in group['versions']:
+                    version = version_raw['version']
+                    preferred = version_raw == group['preferredVersion']
+                    new_group[version] = self.get_resources_for_api_version(prefix, group['name'], version, preferred)
+                groups[prefix][group['name']] = new_group
+            self.__cache['resources'] = groups
+        return self.__cache['resources']
 
     def get_resources_for_api_version(self, prefix, group, version, preferred):
         """ returns a dictionary of resources associated with provided groupVersion"""
@@ -369,6 +423,24 @@ def __init__(self, prefix=None, group=None, api_version=None, kind=None,
 
         self.extra_args = kwargs
 
+    def to_dict(self):
+        return {
+            '_type': 'Resource',
+            'prefix': self.prefix,
+            'group': self.group,
+            'api_version': self.api_version,
+            'kind': self.kind,
+            'namespaced': self.namespaced,
+            'verbs': self.verbs,
+            'name': self.name,
+            'preferred': self.preferred,
+            'singular_name': self.singular_name,
+            'short_names': self.short_names,
+            'categories': self.categories,
+            'subresources': {k: sr.to_dict() for k, sr in self.subresources.items()},
+            'extra_args': self.extra_args,
+        }
+
     @property
     def group_version(self):
         if self.group:
@@ -455,6 +527,12 @@ def patch(self, *args, **kwargs):
     def __getattr__(self, name):
         return getattr(self.resource, name)
 
+    def to_dict(self):
+        return {
+            '_type': 'ResourceList',
+            'resource': self.resource.to_dict(),
+        }
+
 
 class Subresource(Resource):
     """ Represents a subresource of an API resource. This generally includes operations
@@ -498,13 +576,27 @@ def urls(self):
     def __getattr__(self, name):
         return partial(getattr(self.parent.client, name), self)
 
+    def to_dict(self):
+        return {
+            'kind': self.kind,
+            'name': self.name,
+            'subresource': self.subresource,
+            'namespaced': self.namespaced,
+            'verbs': self.verbs,
+            'extra_args': self.extra_args,
+        }
+
 
 class ResourceContainer(object):
     """ A convenient container for storing discovered API resources. Allows
         easy searching and retrieval of specific resources
     """
 
-    def __init__(self, resources):
+    def __init__(self, resources, client=None):
+        self.__resources = resources
+        self.__client = client
+
+    def update(self, resources):
         self.__resources = resources
 
     @property
@@ -544,7 +636,11 @@ def search(self, **kwargs):
 
             The arbitrary arguments can be any valid attribute for an openshift.dynamic.Resource object
         """
-        return self.__search(self.__build_search(**kwargs), self.__resources)
+        results = self.__search(self.__build_search(**kwargs), self.__resources)
+        if not results:
+            self.__client.invalidate_cache()
+            results = self.__search(self.__build_search(**kwargs), self.__resources)
+        return results
 
     def __build_search(self, kind=None, api_version=None, prefix=None, **kwargs):
         if api_version and '/' in api_version: