Merge branch 'main' of https://github.com/mage-os/mirror-commerce-dat…

…a-export
mage-os · Feb 29, 2024 · 6f56a84 · 6f56a84
2 parents bbb4d88 + c89c4a9
commit 6f56a84
Show file tree

Hide file tree

Showing 13 changed files with 91 additions and 24 deletions.
diff --git a/CatalogDataExporter/etc/di.xml b/CatalogDataExporter/etc/di.xml
@@ -242,6 +242,8 @@
                 <item name="websiteCode" xsi:type="string">websiteCode</item>
             </argument>
 
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
+
             <argument name="feedIdentifierMapping" xsi:type="array">
                 <item name="id" xsi:type="string">productId</item>
                 <item name="store_view_code" xsi:type="string">storeViewCode</item>
@@ -284,6 +286,7 @@
                 <item name="feed_data" xsi:type="string">feed_data</item>
                 <item name="is_deleted" xsi:type="string">is_deleted</item>
             </argument>
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
         </arguments>
     </virtualType>
     <virtualType name="Magento\CatalogDataExporter\Model\Indexer\ProductAttributeDataSerializer" type="Magento\DataExporter\Model\Indexer\DataSerializer">
@@ -380,6 +383,7 @@
                 <item name="feed_data" xsi:type="string">feed_data</item>
                 <item name="is_deleted" xsi:type="string">is_deleted</item>
             </argument>
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
         </arguments>
     </virtualType>
     <virtualType name="Magento\CatalogDataExporter\Model\Indexer\CategoryDataSerializer" type="Magento\DataExporter\Model\Indexer\DataSerializer">

diff --git a/DataExporter/Model/Batch/Feed/Generator.php b/DataExporter/Model/Batch/Feed/Generator.php
@@ -139,7 +139,8 @@ public function generate(FeedIndexMetadata $metadata, array $args = []): BatchIt
                     'sourceTableName' => $sourceTableName,
                     'sourceTableKeyColumns' => $sourceTableKeyColumns,
                     'batchLocator' => $batchLocator,
-                    'dateTimeFormat' => $metadata->getDateTimeFormat()
+                    'dateTimeFormat' => $metadata->getDateTimeFormat(),
+                    'isRemovable' => $metadata->isRemovable()
                 ]
             );
         }

diff --git a/DataExporter/Model/Batch/Feed/Iterator.php b/DataExporter/Model/Batch/Feed/Iterator.php
@@ -82,6 +82,13 @@ class Iterator implements BatchIteratorInterface
      */
     private ?string $dateTimeFormat;
 
+    /**
+     * Is removable flag.
+     *
+     * @var bool|null
+     */
+    private ?bool $isRemovable;
+
     /**
      * @var CommerceDataExportLoggerInterface
      */
@@ -95,6 +102,7 @@ class Iterator implements BatchIteratorInterface
      * @param array $sourceTableKeyColumns
      * @param CommerceDataExportLoggerInterface $logger
      * @param string|null $dateTimeFormat
+     * @param bool|null $isRemovable
      */
     public function __construct(
         ResourceConnection $resourceConnection,
@@ -103,7 +111,8 @@ public function __construct(
         string             $sourceTableName,
         array              $sourceTableKeyColumns,
         CommerceDataExportLoggerInterface $logger,
-        ?string            $dateTimeFormat = null
+        ?string            $dateTimeFormat = null,
+        ?bool              $isRemovable = null
     ) {
         $this->resourceConnection = $resourceConnection;
         $this->batchLocator = $batchLocator;
@@ -112,6 +121,7 @@ public function __construct(
         $this->sourceTableKeyColumns = $sourceTableKeyColumns;
         $this->dateTimeFormat = $dateTimeFormat;
         $this->logger = $logger;
+        $this->isRemovable = $isRemovable;
     }
 
     /**
@@ -201,7 +211,6 @@ function ($key) {
                 ['t' => $this->sourceTableName],
                 [
                     't.' . FeedIndexMetadata::FEED_TABLE_FIELD_FEED_DATA,
-                    't.' . FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED,
                     't.' . FeedIndexMetadata::FEED_TABLE_FIELD_MODIFIED_AT
                 ]
             )
@@ -211,12 +220,16 @@ function ($key) {
                 []
             )
             ->where($this->batchTable->getBatchNumberField() . ' = ?', $this->batchNumber);
-
+        if (true === $this->isRemovable) {
+            $select->columns(
+                't.' . FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED
+            );
+        }
         $statement = $connection->query($select);
         $data = [];
         while ($row = $statement->fetch()) {
             $entity = json_decode($row[FeedIndexMetadata::FEED_TABLE_FIELD_FEED_DATA], true);
-            $entity['deleted'] = (bool)$row[FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED];
+            $entity['deleted'] = $this->isRemovable && $row[FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED];
             $entity['modifiedAt'] = $row[FeedIndexMetadata::FEED_TABLE_FIELD_MODIFIED_AT];
             if (null !== $this->dateTimeFormat) {
                 try {

diff --git a/DataExporter/Model/Batch/FeedChangeLog/Generator.php b/DataExporter/Model/Batch/FeedChangeLog/Generator.php
@@ -89,7 +89,7 @@ public function generate(FeedIndexMetadata $metadata, array $args = []): BatchIt
         if ($connection instanceof \Magento\ResourceConnections\DB\Adapter\Pdo\MysqlProxy) {
             $connection->setUseMasterConnection();
         }
-        $viewId = $metadata->getFeedTableName();
+        $viewId = $args['viewId'] ?? $metadata->getFeedTableName();
         $view = $this->viewFactory->create()->load($viewId);
         $sourceTableName = $this->resourceConnection->getTableName($view->getChangelog()->getName());
         $sourceTableField = $view->getChangelog()->getColumnName();

diff --git a/DataExporter/Model/Indexer/FeedIndexMetadata.php b/DataExporter/Model/Indexer/FeedIndexMetadata.php
@@ -185,7 +185,7 @@ public function __construct(
         array $excludeFromHashFields = [],
         bool $exportImmediately = false,
         bool $persistExportedFeed = false,
-        bool $entitiesRemovable = true,
+        bool $entitiesRemovable = false,
         ?string $dateTimeFormat = null,
         ?Config $config = null
     ) {

diff --git a/DataExporter/Model/Indexer/FeedIndexMetadataProviderInterface.php b/DataExporter/Model/Indexer/FeedIndexMetadataProviderInterface.php
@@ -0,0 +1,32 @@
+<?php
+/************************************************************************
+ *
+ * Copyright 2023 Adobe
+ * All Rights Reserved.
+ *
+ * NOTICE: All information contained herein is, and remains
+ * the property of Adobe and its suppliers, if any. The intellectual
+ * and technical concepts contained herein are proprietary to Adobe
+ * and its suppliers and are protected by all applicable intellectual
+ * property laws, including trade secret and copyright laws.
+ * Dissemination of this information or reproduction of this material
+ * is strictly forbidden unless prior written permission is obtained
+ * from Adobe.
+ * ************************************************************************
+ */
+declare(strict_types=1);
+
+namespace Magento\DataExporter\Model\Indexer;
+
+/**
+ * Feed index metadata provider interface
+ */
+interface FeedIndexMetadataProviderInterface
+{
+    /**
+     * Get feed index metadata for the feed
+     *
+     * @return FeedIndexMetadata
+     */
+    public function getFeedIndexMetadata(): FeedIndexMetadata;
+}
diff --git a/DataExporter/Model/Indexer/FeedIndexer.php b/DataExporter/Model/Indexer/FeedIndexer.php
@@ -15,7 +15,7 @@
  * Product export feed indexer class
  * Facade for IndexerProcessor, implements Magento native indexers interfaces
  */
-class FeedIndexer implements IndexerActionInterface, MviewActionInterface
+class FeedIndexer implements IndexerActionInterface, MviewActionInterface, FeedIndexMetadataProviderInterface
 {
     /**
      * @var FeedIndexProcessorCreateUpdate
@@ -118,4 +118,12 @@ public function execute($ids)
             $ids
         );
     }
+
+    /**
+     * @inheritDoc
+     */
+    public function getFeedIndexMetadata(): FeedIndexMetadata
+    {
+        return $this->feedIndexMetadata;
+    }
 }
diff --git a/DataExporter/Model/Indexer/ViewMaterializer.php b/DataExporter/Model/Indexer/ViewMaterializer.php
@@ -22,6 +22,7 @@
 use Magento\DataExporter\Model\FeedPool;
 use Magento\DataExporter\Model\Logging\CommerceDataExportLoggerInterface;
 use Magento\Framework\Mview\ActionFactory;
+use Magento\Framework\Mview\ActionInterface;
 use Magento\Framework\Mview\View\ChangelogTableNotExistsException;
 use Magento\Framework\Mview\View\StateInterface;
 use Magento\Framework\Mview\ViewInterface;
@@ -141,9 +142,9 @@ public function execute(ViewInterface $view): void
     private function executeAction(ViewInterface $view)
     {
         $action = $this->actionFactory->get($view->getActionClass());
-        $feedMetadata = $this->getFeedIndexMetadata($view->getId());
+        $feedMetadata = $this->getFeedIndexMetadata($action);
 
-        $batchIterator = $this->batchGenerator->generate($feedMetadata);
+        $batchIterator = $this->batchGenerator->generate($feedMetadata, ['viewId' => $view->getId()]);
         $threadCount = min($feedMetadata->getThreadCount(), $batchIterator->count());
         $userFunctions = [];
         for ($threadNumber = 1; $threadNumber <= $threadCount; $threadNumber++) {
@@ -171,22 +172,19 @@ private function executeAction(ViewInterface $view)
     }
 
     /**
-     * Returns feed metadata by feed table name.
+     * Returns feed metadata by mview action object
      *
-     * @param string $feedTableName
+     * @param ActionInterface $action
      * @return FeedIndexMetadata
      */
-    private function getFeedIndexMetadata(string $feedTableName): FeedIndexMetadata
+    private function getFeedIndexMetadata(ActionInterface $action): FeedIndexMetadata
     {
-        $feedList = $this->feedPool->getList();
-        foreach ($feedList as $feed) {
-            if ($feed->getFeedMetadata()->getFeedTableName() === $feedTableName) {
-                return $feed->getFeedMetadata();
-            }
+        if ($action instanceof FeedIndexMetadataProviderInterface) {
+            return $action->getFeedIndexMetadata();
+        } else {
+            throw new \InvalidArgumentException(
+                \sprintf('Feed for the "%s" action class is not registered', $action::class)
+            );
         }
-
-        throw new \InvalidArgumentException(
-            \sprintf('Feed with "%s" table name is not registered', $feedTableName)
-        );
     }
 }
diff --git a/InventoryDataExporter/etc/di.xml b/InventoryDataExporter/etc/di.xml
@@ -33,6 +33,8 @@
                 <item name="feed_data" xsi:type="string">feed_data</item>
                 <item name="is_deleted" xsi:type="string">is_deleted</item>
             </argument>
+
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
         </arguments>
     </virtualType>
     <virtualType name="Magento\InventoryDataExporter\Model\Indexer\StockStatusDataSerializer" type="Magento\DataExporter\Model\Indexer\DataSerializer">

diff --git a/ProductPriceDataExporter/etc/di.xml b/ProductPriceDataExporter/etc/di.xml
@@ -32,6 +32,8 @@
                 <item name="customer_group_code" xsi:type="string">customerGroupCode</item>
             </argument>
 
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
+
         </arguments>
     </virtualType>
     <virtualType name="Magento\ProductPriceDataExporter\Model\Indexer\ProductPriceDataSerializer" type="Magento\DataExporter\Model\Indexer\DataSerializer">

diff --git a/ProductReviewDataExporter/etc/di.xml b/ProductReviewDataExporter/etc/di.xml
@@ -20,6 +20,8 @@
                 <item name="feed_data" xsi:type="string">feed_data</item>
                 <item name="is_deleted" xsi:type="string">is_deleted</item>
             </argument>
+
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
         </arguments>
     </virtualType>
 
@@ -81,6 +83,8 @@
                 <item name="feed_data" xsi:type="string">feed_data</item>
                 <item name="is_deleted" xsi:type="string">is_deleted</item>
             </argument>
+
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
         </arguments>
     </virtualType>
 

diff --git a/ProductVariantDataExporter/etc/di.xml b/ProductVariantDataExporter/etc/di.xml
@@ -18,6 +18,8 @@
                 <item name="feed_data" xsi:type="string">feed_data</item>
                 <item name="is_deleted" xsi:type="string">is_deleted</item>
             </argument>
+
+            <argument name="entitiesRemovable" xsi:type="boolean">true</argument>
         </arguments>
     </virtualType>
     <virtualType name="Magento\ProductVariantDataExporter\Model\Indexer\ProductVariantDataSerializer" type="Magento\DataExporter\Model\Indexer\DataSerializer">

diff --git a/README.md b/README.md
@@ -91,8 +91,9 @@ These codes will be saved in the "status" field of the feed table, to keep infor
   </argument>
 
 ### Feed Index Metadata additional parameters:
-- entitiesRemovable -  this parameter handles feed configuration to cover cases when feed entities are not removable. Default value: `true` - feed entities can be removed. For example, `sales order` feed exports Sales Orders that cannot be deleted.
-
+- entitiesRemovable - this parameter handles feed configuration to cover cases when feed entities are not removable. Default value: `false` - feed entities can not be removed.  For example:
+- `sales order` feed export's Sales Orders entities cannot be deleted and `isRemovable` metadata parameter set to false.
+- `product` feed export's Products can be deleted and `isRemovable` metadata parameter *MUST* be set to true, in other case - feed records wouldn't be marked as deleted in the event of entity removal.
 ### Multi-thread data export mode:
 The purpose of this mode is to speed up the export process by splitting the data into batches and processing them in parallel.
 The performance of data export should be aligned with the limit that is defined for a client at consumer side.