Merge pull request #5 from lutraconsulting/download_revisited

download files with multiple sequential requests

Author: varmar05

cli.py

@@ -127,7 +127,6 @@ def status():
         return
 
     click.echo("Current version: {}".format(project_info["version"]))
-    last_version = versions[-1]
     new_versions = [v for v in versions if num_version(v["name"]) > local_version]
     if new_versions:
         click.secho("### Available updates: {}".format(len(new_versions)), fg="magenta")

mergin/client.py

@@ -9,6 +9,7 @@
 from datetime import datetime
 
 this_dir = os.path.dirname(os.path.realpath(__file__))
+CHUNK_SIZE = 10 * 1024 * 1024
 
 try:
     from requests_toolbelt import MultipartEncoder
@@ -26,8 +27,7 @@
         import pytz
         import dateutil.parser
 
-from .utils import save_to_file, generate_checksum, move_file
-from .multipart import MultipartReader, parse_boundary
+from .utils import generate_checksum, move_file, save_to_file
 
 
 class InvalidProject(Exception):
@@ -175,11 +175,14 @@ def _do_request(self, request):
                 raise ClientError(info.get("detail"))
             raise ClientError(e.read().decode("utf-8"))
 
-    def get(self, path, data=None):
+    def get(self, path, data=None, headers={}):
         url = urllib.parse.urljoin(self.url, urllib.parse.quote(path))
         if data:
             url += "?" + urllib.parse.urlencode(data)
-        request = urllib.request.Request(url)
+        if headers:
+            request = urllib.request.Request(url, headers=headers)
+        else:
+            request = urllib.request.Request(url)
         return self._do_request(request)
 
     def post(self, path, data, headers={}):
@@ -341,7 +344,7 @@ def project_versions(self, project_path):
 
     def download_project(self, project_path, directory):
         """
-        Download last version of project into given directory.
+        Download latest version of project into given directory.
 
         :param project_path: Project's full name (<namespace>/<name>)
         :type project_path: String
@@ -353,20 +356,15 @@ def download_project(self, project_path, directory):
             raise Exception("Project directory already exists")
         os.makedirs(directory)
 
-        # TODO: this shouldn't be two independent operations, or we should use version tag in download requests.
-        # But current server API doesn't allow something better.
         project_info = self.project_info(project_path)
-        resp = self.get("/v1/project/download/{}".format(project_path))
-        reader = MultipartReader(resp, parse_boundary(resp.headers["Content-Type"]))
-        part = reader.next_part()
-        while part:
-            dest = os.path.join(directory, part.filename)
-            save_to_file(part, dest)
-            part = reader.next_part()
+        version = project_info['version'] if project_info['version'] else 'v0'
+
+        for file in project_info['files']:
+            self._download_file(project_path, version, file, directory)
 
         data = {
             "name": project_path,
-            "version": project_info["version"],
+            "version": version,
             "files": project_info["files"]
         }
         save_project_file(directory, data)
@@ -453,20 +451,9 @@ def backup_if_conflict(path, checksum):
 
         fetch_files = pull_changes["added"] + pull_changes["updated"]
         if fetch_files:
-            resp = self.post(
-                "/v1/project/fetch/{}".format(project_path),
-                fetch_files,
-                {"Content-Type": "application/json"}
-            )
-            reader = MultipartReader(resp, parse_boundary(resp.headers["Content-Type"]))
-            part = reader.next_part()
             temp_dir = os.path.join(directory, '.mergin', 'fetch_{}-{}'.format(local_info["version"], server_info["version"]))
-            while part:
-                dest = os.path.join(temp_dir, part.filename)
-                save_to_file(part, dest)
-                part = reader.next_part()
-
             for file in fetch_files:
+                self._download_file(project_path, server_info['version'], file, temp_dir)
                 src = os.path.join(temp_dir, file["path"])
                 dest = local_path(file["path"])
                 backup_if_conflict(file["path"], file["checksum"])
@@ -482,5 +469,41 @@ def backup_if_conflict(path, checksum):
             move_file(local_path(file["path"]), local_path(file["new_path"]))
 
         local_info["files"] = server_info["files"]
-        local_info["version"] = server_info["version"]
+        local_info["version"] = server_info["version"] if server_info["version"] else 'v0'
         save_project_file(directory, local_info)
+
+    def _download_file(self, project_path, project_version, file, directory):
+        """
+        Helper to download single project file from server in chunks.
+
+        :param project_path: Project's full name (<namespace>/<name>)
+        :type project_path: String
+        :param project_version: Version of the project (v<n>)
+        :type project_version: String
+        :param file: File metadata item from Project['files']
+        :type file: dict
+        :param directory: Project's directory
+        :type directory: String
+        """
+        query_params = {
+            "file": file['path'],
+            "version": project_version
+        }
+        file_dir = os.path.dirname(os.path.normpath(os.path.join(directory, file['path'])))
+        basename = os.path.basename(file['path'])
+        length = 0
+        count = 0
+        while length < file['size']:
+            range_header = {"Range": "bytes={}-{}".format(length, length + CHUNK_SIZE)}
+            resp = self.get("/v1/project/raw/{}".format(project_path), data=query_params, headers=range_header)
+            if resp.status in [200, 206]:
+                save_to_file(resp, os.path.join(file_dir, basename+".{}".format(count)))
+                length += (CHUNK_SIZE + 1)
+                count += 1
+
+        # merge chunks together
+        with open(os.path.join(file_dir, basename), 'wb') as final:
+            for i in range(count):
+                with open(os.path.join(directory, file['path'] + ".{}".format(i)), 'rb') as chunk:
+                    shutil.copyfileobj(chunk, final)
+                os.remove(os.path.join(directory, file['path'] + ".{}".format(i)))