Merge pull request #31 from eldertek/nightly

Nightly v1.1.3
eldertek · Nov 12, 2023 · 97f81a0 · 97f81a0
2 parents b15b28c + caa7b8e
commit 97f81a0
Show file tree

Hide file tree

Showing 13 changed files with 145 additions and 106 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,8 @@
+## 1.1.3 - 2023-11-12
+### Added
+- Paging of duplicates to avoid to load all duplicates at once
+- Infinite-scrolling to load all duplicates in background
+
 ## 1.1.2 - 2023-11-11
 ### Added
 - Auto-fetch duplicates again when reaching the end of the list

diff --git a/appinfo/info.xml b/appinfo/info.xml
@@ -7,7 +7,7 @@
     <summary lang="fr">Économisez de l’espace en trouvant vos fichiers en doublon</summary>
     <description>Are you tired of sifting through piles of files and folders, only to discover multiple copies of the same content cluttering your storage space?</description>
     <description lang="fr">Vous en avez assez de passer au crible des piles de fichiers et de dossiers pour découvrir que plusieurs copies du même contenu encombrent votre espace de stockage ?</description>
-    <version>1.1.2</version>
+    <version>1.1.3</version>
     <licence>agpl</licence>
     <author mail="[email protected]" >André Théo LAURET</author>
     <namespace>DuplicateFinder</namespace>

diff --git a/appinfo/routes.php b/appinfo/routes.php
@@ -2,7 +2,7 @@
  return [
     'routes' => [
         ['name' => 'page#index', 'url' => '/', 'verb' => 'GET'],
-        ['name' => 'duplicate_api#list', 'url' => '/api/duplicates/{type}', 'verb' => 'GET'],
+        ['name' => 'duplicate_api#list', 'url' => '/api/duplicates/{type}', 'verb' => 'GET', 'requirements' => ['page' => '\d+', 'limit' => '\d+']],
         ['name' => 'duplicate_api#acknowledge', 'url' => '/api/duplicates/acknowledge/{hash}', 'verb' => 'POST'],
         ['name' => 'duplicate_api#unacknowledge', 'url' => '/api/duplicates/unacknowledge/{hash}', 'verb' => 'POST'],
         ['name' => 'duplicate_api#find', 'url' => '/api/duplicates/find', 'verb' => 'POST'],

diff --git a/js/duplicatefinder-main.js b/js/duplicatefinder-main.js
diff --git a/js/duplicatefinder-settings.js b/js/duplicatefinder-settings.js
diff --git a/lib/Controller/DuplicateApiController.php b/lib/Controller/DuplicateApiController.php
@@ -47,36 +47,37 @@ public function __construct(
      * @NoAdminRequired
      * @NoCSRFRequired
      */
-
-    public function list(int $offset = 0, int $limit = 30, string $type = 'unacknowledged'): DataResponse
+    public function list(int $page = 1, int $limit = 30, string $type = 'unacknowledged'): DataResponse
     {
         try {
-            $duplicates = [];
-            switch($type) {
-                case 'all':
-                    $duplicates = $this->fileDuplicateService->findAll("all", $this->getUserId(), $limit, $offset, true);
-                    break;
-                case 'acknowledged':
-                    $duplicates = $this->fileDuplicateService->findAll("acknowledged", $this->getUserId(), $limit, $offset, true);
-                    break;
-                case 'unacknowledged':
-                    $duplicates = $this->fileDuplicateService->findAll("unacknowledged", $this->getUserId(), $limit, $offset, true);
-                    break;
-                default:
-                    return new DataResponse(['status' => 'error', 'message' => 'Invalid type']);
-            }
-            return new DataResponse(['status' => 'success', 'data' => $duplicates]);
+            $offset = ($page - 1) * $limit; // Calculate the offset based on the current page and limit
+            $duplicates = $this->fileDuplicateService->findAll($type, $this->getUserId(), $page, $limit, true);
+            $totalItems = $this->fileDuplicateService->getTotalCount($type); 
+            $totalPages = ceil($totalItems / $limit);
+
+            $data = [
+                'status' => 'success',
+                'entities' => $duplicates['entities'],
+                'pagination' => [
+                    'currentPage' => $page,
+                    'totalPages' => $totalPages,
+                    'totalItems' => $totalItems,
+                    'limit' => $limit
+                ]
+            ];
+            return new DataResponse($data);
         } catch (\Exception $e) {
-            $this->logger->error('A unknown exception occured', ['app' => Application::ID, 'exception' => $e]);
+            $this->logger->error('A unknown exception occurred', ['app' => Application::ID, 'exception' => $e]);
             return new DataResponse(['status' => 'error', 'message' => $e->getMessage()]);
         }
     }
 
+
     /**
      * @NoAdminRequired
      * @NoCSRFRequired
      */
-    public function acknowledge(string $hash): DataResponse 
+    public function acknowledge(string $hash): DataResponse
     {
         $this->fileDuplicateMapper->markAsAcknowledged($hash);
         return new DataResponse(['status' => 'success']);
@@ -86,7 +87,7 @@ public function acknowledge(string $hash): DataResponse
      * @NoAdminRequired
      * @NoCSRFRequired
      */
-    public function unacknowledge(string $hash): DataResponse 
+    public function unacknowledge(string $hash): DataResponse
     {
         $this->fileDuplicateMapper->unmarkAcknowledged($hash);
         return new DataResponse(['status' => 'success']);
@@ -100,7 +101,7 @@ public function clear(): DataResponse
         try {
             $this->fileDuplicateService->clear();
             $this->fileInfoService->clear();
-            return new DataResponse(['status'=> 'success']);
+            return new DataResponse(['status' => 'success']);
         } catch (\Exception $e) {
             $this->logger->error('A unknown exception occured', ['app' => Application::ID, 'exception' => $e]);
             return new DataResponse(['status' => 'error', 'message' => $e->getMessage()]);
@@ -116,7 +117,7 @@ public function find(): DataResponse
             $this->userManager->callForAllUsers(function (IUser $user): void {
                 $this->fileInfoService->scanFiles($user->getUID());
             });
-            return new DataResponse(['status'=> 'success']);
+            return new DataResponse(['status' => 'success']);
         } catch (\Exception $e) {
             $this->logger->error('A unknown exception occured', ['app' => Application::ID, 'exception' => $e]);
             return new DataResponse(['status' => 'error', 'message' => $e->getMessage()]);

diff --git a/lib/Db/FileDuplicateMapper.php b/lib/Db/FileDuplicateMapper.php
@@ -47,13 +47,16 @@ public function findAll(
         $qb = $this->db->getQueryBuilder();
         $qb->select('d.id as id', 'type', 'hash', 'acknowledged')
             ->from($this->getTableName(), 'd');
+
         if ($limit !== null) {
-            $qb->setMaxResults($limit);
+            $qb->setMaxResults($limit); // Set the limit of rows to fetch
         }
         if ($offset !== null) {
-            $qb->where($qb->expr()->gt('id', $qb->createNamedParameter($offset, IQueryBuilder::PARAM_INT)));
+            $qb->setFirstResult($offset); // Set the offset to start fetching rows
         }
+
         $qb->addOrderBy('id');
+
         if ($orderBy !== null) {
             foreach ($orderBy as $order) {
                 $qb->addOrderBy($order[0], isset($order[1]) ? $order[1] : null);
@@ -114,5 +117,34 @@ public function unmarkAcknowledged(string $hash): bool
             return false;
         }
     }
-
+
+    /**
+     * Gets the total count of duplicates based on the type.
+     *
+     * @param string $type The type of duplicates to count.
+     * @return int The total count of duplicates.
+     */
+    public function getTotalCount(string $type = 'unacknowledged'): int
+    {
+        $qb = $this->db->getQueryBuilder();
+
+        // Start with a basic SELECT COUNT query
+        $qb->select($qb->func()->count('*', 'total_count'))
+            ->from($this->getTableName());
+
+        // Add conditions based on the type
+        if ($type === 'acknowledged') {
+            $qb->where($qb->expr()->eq('acknowledged', $qb->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)));
+        } elseif ($type === 'unacknowledged') {
+            $qb->where($qb->expr()->eq('acknowledged', $qb->createNamedParameter(false, IQueryBuilder::PARAM_BOOL)));
+        } // No condition needed for 'all', as we want to count all rows
+
+        // Execute the query and fetch the result
+        $result = $qb->execute();
+        $row = $result->fetch();
+        $result->closeCursor();
+
+        // Return the count result as an integer
+        return (int) ($row ? $row['total_count'] : 0);
+    }
 }
diff --git a/lib/Service/FileDuplicateService.php b/lib/Service/FileDuplicateService.php
@@ -57,11 +57,14 @@ public function enrich(FileDuplicate $duplicate): FileDuplicate
     public function findAll(
         ?string $type = null,
         ?string $user = null,
-        ?int $limit = 20,
-        ?int $offset = null,
+        int $page = 1,
+        int $pageSize = 20,
         bool $enrich = false,
         ?array $orderBy = [['hash'], ['type']]
     ): array {
+        $limit = $pageSize; // Set the number of records per page
+        $offset = ($page - 1) * $pageSize; // Calculate the offset
+
         $result = array();
         $entities = null;
         do {
@@ -155,4 +158,9 @@ public function clear(): void
     {
         $this->mapper->clear();
     }
+
+    public function getTotalCount(string $type = 'unacknowledged'): int
+    {
+        return $this->mapper->getTotalCount($type);
+    }
 }
diff --git a/lib/Service/FileInfoService.php b/lib/Service/FileInfoService.php
@@ -3,7 +3,6 @@
 namespace OCA\DuplicateFinder\Service;
 
 use Psr\Log\LoggerInterface;
-use OCP\IDBConnection;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\Node;
 use OCP\Files\NotFoundException;
@@ -285,7 +284,7 @@ public function hasAccessRight(FileInfo $fileInfo, string $user): ?FileInfo
                     $fileInfo->setPath($path);
                     $result = $fileInfo;
                 }
-            } catch (NoUserException | NotFoundException $e) {
+            } catch (NotFoundException $e) {
                 $result = null;
             }
         }

diff --git a/lib/Utils/CMDUtils.php b/lib/Utils/CMDUtils.php
@@ -11,56 +11,67 @@ public static function showDuplicates(
         FileDuplicateService $fileDuplicateService,
         OutputInterface $output,
         \Closure $abortIfInterrupted,
-        ?string $user = null
+        ?string $user = null,
+        int $pageSize = 20 // Add a page size parameter
     ): void {
-        $output->writeln($user === null ? 'Duplicates are: ' : 'Duplicates for user "'.$user.'" are: ');
-        $duplicates = array("pageKey" => 0, "isLastFetched" => true);
+        $output->writeln($user === null ? 'Duplicates are: ' : 'Duplicates for user "' . $user . '" are: ');
+
+        $currentPage = 1; // Start from the first page
+        $isLastFetched = false;
+
         do {
-            $duplicates = $fileDuplicateService->findAll('all', $user, 20, $duplicates["pageKey"], true);
+            // Pass the current page and page size to the findAll method
+            $duplicates = $fileDuplicateService->findAll('all', $user, $currentPage, $pageSize, true);
+
             self::processDuplicates($output, $duplicates);
             $abortIfInterrupted();
-        } while (!$duplicates["isLastFetched"]);
+
+            $isLastFetched = $duplicates["isLastFetched"];
+            $currentPage++; // Increment to fetch the next page in the next iteration
+        } while (!$isLastFetched); // Continue until the last page is fetched
     }
 
-    private static function processDuplicates(OutputInterface $output, array $duplicates): void {
+
+    private static function processDuplicates(OutputInterface $output, array $duplicates): void
+    {
         foreach ($duplicates["entities"] as $duplicate) {
             if (!$duplicate->getFiles()) {
                 continue;
             }
-            $output->writeln($duplicate->getHash().'('.$duplicate->getType().')');
+            $output->writeln($duplicate->getHash() . '(' . $duplicate->getType() . ')');
             self::showFiles($output, $duplicate->getFiles());
         }
     }
 
     /**
      * @param array<\OCA\DuplicateFinder\Db\FileInfo> $files
      */
-    private static function showFiles(OutputInterface $output, array $files) : void
+    private static function showFiles(OutputInterface $output, array $files): void
     {
         $shownPaths = [];
         $hiddenPaths = 0;
         $indent = '     ';
         foreach ($files as $file) {
             if ($file instanceof \OCA\DuplicateFinder\Db\FileInfo) {
                 if (!isset($shownPaths[$file->getPath()])) {
-                    $output->writeln($indent.$file->getPath());
+                    $output->writeln($indent . $file->getPath());
                     $shownPaths[$file->getPath()] = 1;
                 } else {
                     $hiddenPaths += 1;
                 }
             }
         }
         if ($hiddenPaths > 0) {
-            $message = $hiddenPaths.' path'.($hiddenPaths > 1 ? 's are' : ' is').' hidden because '.($hiddenPaths > 1 ? 'they reference' : 'it references').' to a similiar file.';
-            $output->writeln($indent.'<info>'.$message.'</info>');
+            $message = $hiddenPaths . ' path' . ($hiddenPaths > 1 ? 's are' : ' is') . ' hidden because ' . ($hiddenPaths > 1 ? 'they reference' : 'it references') . ' to a similiar file.';
+            $output->writeln($indent . '<info>' . $message . '</info>');
         }
     }
 
     public static function showIfOutputIsPresent(
         string $message,
         ?OutputInterface $output = null,
         int $verbosity = OutputInterface::VERBOSITY_NORMAL
-    ) : void {
+    ): void {
         if (!is_null($output)) {
             $output->writeln($message, $verbosity);
         }

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "duplicatefinder",
   "description": "Save some space by finding your duplicate files",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "author": "André Théo LAURET <[email protected]>",
   "contributors": [],
   "bugs": {