Add Checksum Verification and Directory Upload Support for QFieldCloud

src/core/utils/qfieldcloudutils.cpp

@@ -13,7 +13,9 @@
  *                                                                         *
  ***************************************************************************/
 
+#include "fileutils.h"
 #include "platformutilities.h"
+#include "qfieldcloudconnection.h"
 #include "qfieldcloudutils.h"
 #include "stringutils.h"
 
@@ -150,26 +152,110 @@ const QMultiMap<QString, QString> QFieldCloudUtils::getPendingAttachments()
   return std::move( files );
 }
 
-void QFieldCloudUtils::addPendingAttachments( const QString &projectId, const QStringList &fileNames )
+void QFieldCloudUtils::addPendingAttachments( const QString &projectId, const QStringList &fileNames, QFieldCloudConnection *cloudConnection, const bool &checkSumCheck )
+{
+  if ( checkSumCheck && cloudConnection )
+  {
+    QVariantMap params;
+    params.insert( "skip_metadata", 1 );
+    NetworkReply *reply = cloudConnection->get( QStringLiteral( "/api/v1/files/%1/" ).arg( projectId ), params );
+
+    connect( reply, &NetworkReply::finished, reply, [=]() {
+      QNetworkReply *rawReply = reply->currentRawReply();
+      reply->deleteLater();
+
+      if ( rawReply->error() != QNetworkReply::NoError )
+      {
+        QgsLogger::debug( QStringLiteral( "Project %1: failed to retrieve file information. %2" ).arg( projectId, QFieldCloudConnection::errorString( rawReply ) ) );
+        return;
+      }
+
+      const QJsonArray files = QJsonDocument::fromJson( rawReply->readAll() ).array();
+      QHash<QString, QString> fileChecksumMap;
+
+      for ( const QJsonValueConstRef &fileValue : files )
+      {
+        const QJsonObject fileObject = fileValue.toObject();
+        const QString cloudEtag = fileObject.value( QStringLiteral( "md5sum" ) ).toString();
+        const QString fileName = fileObject.value( QStringLiteral( "name" ) ).toString();
+        fileChecksumMap.insert( fileName, cloudEtag );
+      }
+
+      writeToAttachmentsFile( projectId, fileNames, &fileChecksumMap, checkSumCheck );
+    } );
+  }
+  else
+  {
+    writeToAttachmentsFile( projectId, fileNames, nullptr, false );
+  }
+}
+
+void QFieldCloudUtils::writeToAttachmentsFile( const QString &projectId, const QStringList &fileNames, const QHash<QString, QString> *fileChecksumMap, const bool &checkSumCheck )
 {
   QLockFile attachmentsLock( QStringLiteral( "%1/attachments.lock" ).arg( QFieldCloudUtils::localCloudDirectory() ) );
   if ( attachmentsLock.tryLock( 10000 ) )
   {
     QFile attachmentsFile( QStringLiteral( "%1/attachments.csv" ).arg( QFieldCloudUtils::localCloudDirectory() ) );
     attachmentsFile.open( QFile::Append | QFile::Text );
     QTextStream attachmentsStream( &attachmentsFile );
-    QFileInfo fi( attachmentsFile );
 
     for ( const QString &fileName : fileNames )
     {
-      QStringList values = QStringList() << projectId << fileName;
-      attachmentsStream << StringUtils::stringListToCsv( values )
-                        << Qt::endl;
+      QFileInfo fi( QDir::cleanPath( fileName ) );
+      if ( fi.isDir() )
+      {
+        writeFilesFromDirectory( fileName, projectId, fileChecksumMap, checkSumCheck, attachmentsStream );
+      }
+      else if ( fi.isFile() )
+      {
+        writeFileDetails( fileName, projectId, fileChecksumMap, checkSumCheck, attachmentsStream );
+      }
     }
+
     attachmentsFile.close();
   }
 }
 
+void QFieldCloudUtils::writeFilesFromDirectory( const QString &dirPath, const QString &projectId, const QHash<QString, QString> *fileChecksumMap, const bool &checkSumCheck, QTextStream &attachmentsStream )
+{
+  QDir dir( dirPath );
+  if ( !dir.exists() )
+  {
+    return;
+  }
+
+  QFileInfoList entries = dir.entryInfoList( QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot );
+
+  for ( const QFileInfo &entry : entries )
+  {
+    if ( entry.isDir() )
+    {
+      writeFilesFromDirectory( entry.absoluteFilePath(), projectId, fileChecksumMap, checkSumCheck, attachmentsStream );
+    }
+    else if ( entry.isFile() )
+    {
+      writeFileDetails( entry.absoluteFilePath(), projectId, fileChecksumMap, checkSumCheck, attachmentsStream );
+    }
+  }
+}
+
+void QFieldCloudUtils::writeFileDetails( const QString &fileName, const QString &projectId, const QHash<QString, QString> *fileChecksumMap, const bool &checkSumCheck, QTextStream &attachmentsStream )
+{
+  const QString localEtag = FileUtils::fileEtag( fileName );
+  QString cloudFileName = "";
+  const QStringList fileNameParts = fileName.split( projectId + "/" );
+  if ( fileNameParts.size() > 1 )
+  {
+    cloudFileName = fileNameParts[1];
+  }
+
+  if ( !checkSumCheck || localEtag != fileChecksumMap->value( cloudFileName ) )
+  {
+    QStringList values { projectId, fileName };
+    attachmentsStream << StringUtils::stringListToCsv( values ) << Qt::endl;
+  }
+}
+
 void QFieldCloudUtils::removePendingAttachment( const QString &projectId, const QString &fileName )
 {
   QLockFile attachmentsLock( QStringLiteral( "%1/attachments.lock" ).arg( QFieldCloudUtils::localCloudDirectory() ) );

src/core/utils/qfieldcloudutils.h

@@ -1,5 +1,5 @@
 /***************************************************************************
-    qfieldcloudutils.cpp
+    qfieldcloudutils.h
     ---------------------
     begin                : February 2020
     copyright            : (C) 2020 by Mathieu Pellerin
@@ -134,14 +134,28 @@ class QFieldCloudUtils : public QObject
     //! Returns the list of attachments that have not yet been uploaded to the cloud.
     static const QMultiMap<QString, QString> getPendingAttachments();
 
-    //! Adds an array of \a fileNames for a given \a projectId to the pending attachments list
-    Q_INVOKABLE static void addPendingAttachments( const QString &projectId, const QStringList &fileNames );
+    /**
+     * Adds an array of files and/or folders for a given cloud project to the pending upload attachments list.
+     * If \a checkSumCheck is true, checks file checksums with the server; otherwise, adds all files without validation.
+     *
+     * @param projectId The project ID for which files are added.
+     * @param fileNames The list of file and/or folder path(s) to be added.
+     * @param cloudConnection The cloud connection used to fetch file data.
+     * @param checkSumCheck Whether to validate files by comparing checksums with the server.
+     */
+    Q_INVOKABLE static void addPendingAttachments( const QString &projectId, const QStringList &fileNames, QFieldCloudConnection *cloudConnection = nullptr, const bool &checkSumCheck = false );
 
     //! Adds removes a \a fileName for a given \a projectId to the pending attachments list
     static void removePendingAttachment( const QString &projectId, const QString &fileName );
 
   private:
     static inline const QString errorCodeOverQuota { QStringLiteral( "over_quota" ) };
+
+    static void writeToAttachmentsFile( const QString &projectId, const QStringList &fileNames, const QHash<QString, QString> *fileChecksumMap, const bool &checkSumCheck );
+
+    static void writeFilesFromDirectory( const QString &dirPath, const QString &projectId, const QHash<QString, QString> *fileChecksumMap, const bool &checkSumCheck, QTextStream &attachmentsStream );
+
+    static void writeFileDetails( const QString &fileName, const QString &projectId, const QHash<QString, QString> *fileChecksumMap, const bool &checkSumCheck, QTextStream &attachmentsStream );
 };
 
 #endif // QFIELDCLOUDUTILS_H

src/qml/QFieldLocalDataPickerScreen.qml

@@ -160,6 +160,7 @@ Page {
           property string itemPath: ItemPath
           property bool itemIsFavorite: ItemIsFavorite
           property bool itemChecked: ItemChecked
+          property bool itemWithinQFieldCloudProjectFolder: cloudProjectsModel.currentProjectId !== "" && itemPath.search(cloudProjectsModel.currentProjectId) !== -1
           property bool itemHasWebdavConfiguration: ItemHasWebdavConfiguration
           property bool itemMenuLoadable: !projectFolderView && (ItemMetaType === LocalFilesModel.Project || ItemMetaType === LocalFilesModel.Dataset)
           property bool itemMenuVisible: ((ItemType === LocalFilesModel.SimpleFolder || ItemMetaType == LocalFilesModel.Dataset || ItemMetaType == LocalFilesModel.File) && table.model.currentPath !== 'root') || ((platformUtilities.capabilities & PlatformUtilities.CustomExport || platformUtilities.capabilities & PlatformUtilities.CustomSend) && (ItemMetaType === LocalFilesModel.Dataset)) || (ItemMetaType === LocalFilesModel.Dataset && ItemType === LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId)
@@ -285,6 +286,7 @@ Page {
                 itemMenu.itemType = ItemType;
                 itemMenu.itemPath = ItemPath;
                 itemMenu.itemIsFavorite = ItemIsFavorite;
+                itemMenu.itemWithinQFieldCloudProjectFolder = rectangle.itemWithinQFieldCloudProjectFolder;
                 itemMenu.itemHasWebdavConfiguration = ItemHasWebdavConfiguration;
                 itemMenu.popup(gc.x + width - itemMenu.width, gc.y - height);
               }
@@ -303,7 +305,7 @@ Page {
             for (let i = 0; i < table.selectedList.length; ++i) {
               const item = table.itemAtIndex(table.selectedList[i]);
               table.selectedItemsWebDavConfigured = table.selectedItemsWebDavConfigured && item.itemHasWebdavConfiguration;
-              table.selectedItemsPushableToQField = table.selectedItemsPushableToQField && item.itemMetaType == LocalFilesModel.Dataset && item.itemType == LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId;
+              table.selectedItemsPushableToQField = (table.selectedItemsPushableToQField && item.itemMetaType == LocalFilesModel.Dataset && item.itemType == LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId) || (item.itemMetaType == LocalFilesModel.Folder && item.itemWithinQFieldCloudProjectFolder);
             }
           }
         }
@@ -434,6 +436,7 @@ Page {
       property string itemPath: ''
       property bool itemIsFavorite: false
       property bool itemHasWebdavConfiguration: false
+      property bool itemWithinQFieldCloudProjectFolder: false
 
       title: qsTr('Item Actions')
 
@@ -460,7 +463,7 @@ Page {
 
       MenuItem {
         id: pushDatasetToCloud
-        enabled: itemMenu.itemMetaType == LocalFilesModel.Dataset && itemMenu.itemType == LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId
+        enabled: (itemMenu.itemMetaType == LocalFilesModel.Dataset && itemMenu.itemType == LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId) || (itemMenu.itemMetaType == LocalFilesModel.Folder && itemMenu.itemWithinQFieldCloudProjectFolder)
         visible: enabled
 
         font: Theme.defaultFont
@@ -470,7 +473,7 @@ Page {
 
         text: qsTr("Push to QFieldCloud")
         onTriggered: {
-          QFieldCloudUtils.addPendingAttachments(cloudProjectsModel.currentProjectId, [itemMenu.itemPath]);
+          QFieldCloudUtils.addPendingAttachments(cloudProjectsModel.currentProjectId, [itemMenu.itemPath], cloudConnection, true);
           platformUtilities.uploadPendingAttachments(cloudConnection);
           displayToast(qsTr("‘%1’ is being uploaded to QFieldCloud").arg(FileUtils.fileName(itemMenu.itemPath)));
         }
@@ -852,13 +855,13 @@ Page {
           var fileNames = [];
           for (let i = 0; i < table.selectedList.length; ++i) {
             const item = table.itemAtIndex(table.selectedList[i]);
-            const pushableToCloud = item.itemMetaType == LocalFilesModel.Dataset && item.itemType == LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId;
+            const pushableToCloud = (item.itemMetaType == LocalFilesModel.Dataset && item.itemType == LocalFilesModel.RasterDataset && cloudProjectsModel.currentProjectId) || (item.itemMetaType == LocalFilesModel.Folder && item.itemWithinQFieldCloudProjectFolder);
             if (pushableToCloud) {
               fileNames.push(item.itemPath);
             }
           }
           if (fileNames.length > 0) {
-            QFieldCloudUtils.addPendingAttachments(cloudProjectsModel.currentProjectId, fileNames);
+            QFieldCloudUtils.addPendingAttachments(cloudProjectsModel.currentProjectId, fileNames, cloudConnection, true);
             platformUtilities.uploadPendingAttachments(cloudConnection);
             localFilesModel.clearSelection();
           } else {