Merge pull request #81 from Nyholm/timeout

Better timeout options

Client/GoogleDriveClient.php

@@ -1,4 +1,5 @@
 <?php
+
 namespace Dizda\CloudBackupBundle\Client;
 
 use Happyr\GoogleSiteAuthenticatorBundle\Service\ClientProvider;
@@ -10,6 +11,8 @@
  */
 class GoogleDriveClient implements ClientInterface
 {
+    const CHUNK_SIZE_BYTES = 20971520; //20 * 1024 * 1024
+
     /**
      * @var \Happyr\GoogleSiteAuthenticatorBundle\Service\ClientProvider clientProvider
      */
@@ -20,16 +23,28 @@ class GoogleDriveClient implements ClientInterface
      */
     protected $remotePath;
 
+    /**
+     * @var int
+     */
+    protected $timeout;
+
+    /**
+     * @var \Google_Client
+     */
+    private $client;
+
     /**
      * @param ClientProvider $clientProvider
      * @param string         $tokenName
      * @param string         $remotePath
+     * @param int            $timeout
      */
-    public function __construct(ClientProvider $clientProvider, $tokenName, $remotePath)
+    public function __construct(ClientProvider $clientProvider, $tokenName, $remotePath, $timeout)
     {
         $this->clientProvider = $clientProvider;
         $this->tokenName = $tokenName;
         $this->remotePath = $remotePath;
+        $this->timeout = $timeout;
     }
 
     /**
@@ -98,7 +113,15 @@ protected function getParentFolder(\Google_Service_Drive $service)
      */
     protected function getDriveService()
     {
-        $client = $this->clientProvider->getClient($this->tokenName);
+        $client = $this->getClient();
+
+        // Make sure CURL do not timeout on you when you upload a large file
+        $client->setClassConfig('Google_IO_Curl', 'options',
+            array(
+                CURLOPT_TIMEOUT => $this->timeout,
+            )
+        );
+
         $service = new \Google_Service_Drive($client);
 
         return $service;
@@ -120,30 +143,14 @@ protected function getDriveFile($archive)
     }
 
     /**
-     * @param $archive
-     *
-     * @return string
-     */
-    protected function getFileContents($archive)
-    {
-        $data = @file_get_contents($archive);
-
-        if ($data === false) {
-            throw new \Exception(sprintf('Could not read file: %s', $archive));
-        }
-
-        return $data;
-    }
-
-    /**
-     * @param $service
-     * @param $archive
+     * @param \Google_Service_Drive $service
+     * @param string                $archive
      *
-     * @return mixed
+     * @return \Google_Service_Drive_DriveFile
      *
      * @throws \Exception
      */
-    private function handleUpload($service, $archive)
+    private function handleUpload(\Google_Service_Drive $service, $archive)
     {
         $file = $this->getDriveFile($archive);
 
@@ -158,11 +165,43 @@ private function handleUpload($service, $archive)
             }
         }
 
-        return $service->files->insert($file, array(
-            'data' => $this->getFileContents($archive),
-            'mimeType' => $mime,
-            'uploadType' => 'media',
-        ));
+        $client = $this->getClient();
+        // Call the API with the media upload, defer so it doesn't immediately return.
+        $client->setDefer(true);
+        $request = $service->files->insert($file);
+
+        // Create a media file upload to represent our upload process.
+        $media = $this->getMediaUploadFile($archive, $client, $request, $mime);
+
+        return $this->uploadFileInChunks($archive, $media);
+    }
+
+    /**
+     * Get a chunk of data from a file resource
+     *
+     * @param resource $handle
+     * @param integer $chunkSize
+     *
+     * @return string
+     */
+    protected function readChunk($handle, $chunkSize)
+    {
+        $byteCount = 0;
+        $giantChunk = '';
+        while (!feof($handle)) {
+            /*
+             * fread will never return more than 8192 bytes if the stream is read
+             * buffered and it does not represent a plain file
+             */
+            $chunk = fread($handle, 8192);
+            $byteCount += strlen($chunk);
+            $giantChunk .= $chunk;
+            if ($byteCount >= $chunkSize) {
+                return $giantChunk;
+            }
+        }
+
+        return $giantChunk;
     }
 
     /**
@@ -172,4 +211,51 @@ public function getName()
     {
         return 'GoogleDrive';
     }
+
+    /**
+     * @return \Google_Client
+     */
+    protected function getClient()
+    {
+        if ($this->client === null) {
+            $this->client = $this->clientProvider->getClient($this->tokenName);
+        }
+
+        return $this->client;
+    }
+
+    /**
+     * @param $archive
+     * @param $media
+     *
+     * @return \Google_Service_Drive_DriveFile
+     */
+    protected function uploadFileInChunks($archive, $media)
+    {
+        $status = false;
+        $handle = fopen($archive, 'rb');
+        while (!$status && !feof($handle)) {
+            $chunk = $this->readChunk($handle, self::CHUNK_SIZE_BYTES);
+            $status = $media->nextChunk($chunk);
+        }
+        fclose($handle);
+
+        return $status;
+    }
+
+    /**
+     * @param $archive
+     * @param $client
+     * @param $request
+     * @param $mime
+     *
+     * @return \Google_Http_MediaFileUpload
+     */
+    protected function getMediaUploadFile($archive, $client, $request, $mime)
+    {
+        $media = new \Google_Http_MediaFileUpload($client, $request, $mime, null, true, self::CHUNK_SIZE_BYTES);
+        $media->setFileSize(filesize($archive));
+
+        return $media;
+    }
 }

Resources/config/services.yml

@@ -1,3 +1,6 @@
+parameters:
+    dizda.cloudbackup.google_drive.timeout: 100
+
 services:
     dizda.cloudbackup.manager.database:
         class: Dizda\CloudBackupBundle\Manager\DatabaseManager
@@ -40,7 +43,7 @@ services:
     dizda.cloudbackup.client.google_drive:
         class: Dizda\CloudBackupBundle\Client\GoogleDriveClient
         public: false
-        arguments: [~, ~, ~]
+        arguments: [~, ~, ~, %dizda.cloudbackup.google_drive.timeout%]
         tags:
             -  { name: dizda.cloudbackup.client }
 

Tests/Client/GoogleDriveClientTest.php

@@ -17,6 +17,14 @@ public function testUpload()
         $driveParent = $this->getMock('Google_Service_Drive_ParentReference');
         $driveService = $this->getDriveService();
 
+        $client = $this->getMockBuilder('Google_Client')
+            ->disableOriginalConstructor()
+            ->setMethods(array('setDefer'))
+            ->getMock();
+
+        $client->expects($this->once())
+            ->method('setDefer');
+
         $driveFile = $this->getMock('Google_Service_Drive_DriveFile');
         $driveFile->expects($this->once())
             ->method('setMimeType')
@@ -27,8 +35,8 @@ public function testUpload()
             ->with($this->equalTo(array($driveParent)));
 
         $drive = $this->getMockBuilder('Dizda\CloudBackupBundle\Client\GoogleDriveClient')
-            ->setConstructorArgs(array($clientProvider, 'foobar', '/foo/bar'))
-            ->setMethods(array('output', 'getDriveService', 'getDriveFile', 'getMimeType', 'getParentFolder', 'getFileContents'))
+            ->setConstructorArgs(array($clientProvider, 'foobar', '/foo/bar', '100'))
+            ->setMethods(array('getClient', 'uploadFileInChunks', 'getMediaUploadFile', 'getDriveService', 'getDriveFile', 'getMimeType', 'getParentFolder'))
             ->getMock();
 
         $drive->expects($this->any())
@@ -54,9 +62,18 @@ public function testUpload()
             ->willReturn($driveParent);
 
         $drive->expects($this->once())
-            ->method('getFileContents')
-            ->with($this->equalTo($archive))
-            ->willReturn('data');
+            ->method('getClient')
+            ->willReturn($client);
+
+        $drive->expects($this->once())
+            ->method('getMediaUploadFile')
+            ->with($this->equalTo($archive), $this->equalTo($client), $this->equalTo('request'), $this->equalTo($mime))
+            ->willReturn('media');
+
+        $drive->expects($this->once())
+            ->method('uploadFileInChunks')
+            ->with($this->equalTo($archive), $this->equalTo('media'))
+            ->willReturn('media');
 
         $drive->upload($archive);
     }
@@ -71,7 +88,8 @@ private function getDriveService()
             ->setMethods(array('insert'))
             ->getMock();
         $driveFiles->expects($this->once())
-            ->method('insert');
+            ->method('insert')
+            ->willReturn('request');
 
         $driveService = $this->getMockBuilder('Google_Service_Drive')
             ->disableOriginalConstructor()
@@ -82,14 +100,6 @@ private function getDriveService()
         return $driveService;
     }
 
-    /**
-     * @expectedException \Exception
-     */
-    public function testGetFileContents()
-    {
-        $drive = new Dummy();
-        $drive->getFileContents('/example/path');
-    }
 
     public function testGetDriveFile()
     {
@@ -122,12 +132,4 @@ public function getDriveFile($a)
     {
         return parent::getDriveFile($a);
     }
-    public function output($a, $b = true)
-    {
-        parent::output($a, $b);
-    }
-    public function getFileContents($a)
-    {
-        return parent::getFileContents($a);
-    }
 }