Merge branch 'main' of https://github.com/mage-os/mirror-commerce-dat…

…a-export

BundleProductDataExporter/etc/mview.xml

@@ -6,7 +6,7 @@
  */
 -->
 <config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:framework:Mview/etc/mview.xsd">
-    <view id="catalog_data_exporter_products" class="Magento\CatalogDataExporter\Model\Indexer\ProductFeedIndexer" group="indexer">
+    <view id="cde_products_feed" class="Magento\CatalogDataExporter\Model\Indexer\ProductFeedIndexer" group="indexer">
         <subscriptions>
             <!-- used to collect option "type" and "required" attributes-->
             <table name="catalog_product_bundle_option" entity_column="parent_id" />

CatalogDataExporter/Model/Provider/Categories.php

@@ -12,14 +12,16 @@
 use Magento\CatalogDataExporter\Model\Provider\EavAttributes\EntityEavAttributesResolver;
 use Magento\CatalogDataExporter\Model\Query\CategoryMainQuery;
 use Magento\DataExporter\Exception\UnableRetrieveData;
+use Magento\DataExporter\Export\DataProcessorInterface;
+use Magento\DataExporter\Model\Indexer\FeedIndexMetadata;
 use Magento\Framework\App\ResourceConnection;
 use Magento\Store\Model\Store;
 use Magento\DataExporter\Model\Logging\CommerceDataExportLoggerInterface as LoggerInterface;
 
 /**
  * Categories main data provider
  */
-class Categories
+class Categories implements DataProcessorInterface
 {
     /**
      * @var ResourceConnection
@@ -68,52 +70,92 @@ public function __construct(
     }
 
     /**
-     * Get provider data
+     * @inheritdoc
      *
-     * @param array $values
+     * @throws UnableRetrieveData
+     * @throws \Zend_Db_Statement_Exception
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
+     */
+    public function execute(
+        array $arguments,
+        callable $dataProcessorCallback,
+        FeedIndexMetadata $metadata,
+        $node = null,
+        $info = null,
+        $lastChunk = null
+    ): void {
+        try {
+            foreach ($this->getDataBatch($arguments, $metadata->getBatchSize()) as $dataBatch) {
+                $output = [];
+                list($mappedCategories, $attributesData) = $dataBatch;
+                $modifiedAt = (new \DateTime())->format($metadata->getDateTimeFormat());
+                foreach ($mappedCategories as $storeCode => $categories) {
+                    $output[] = \array_map(function ($row) use ($modifiedAt) {
+                        $row['modifiedAt'] = $modifiedAt;
+                        return $this->formatter->format($row);
+                    }, \array_replace_recursive(
+                        $categories,
+                        $this->entityEavAttributesResolver->resolve($attributesData[$storeCode], $storeCode)
+                    ));
+                }
+                $dataProcessorCallback($this->get(\array_merge(...$output)));
+            }
+        } catch (\Throwable $exception) {
+            $this->logger->error($exception->getMessage(), ['exception' => $exception]);
+            throw new UnableRetrieveData('Unable to retrieve category data');
+        }
+    }
+
+    /**
+     * For backward compatibility - to allow 3rd party plugins work
      *
+     * @param array $values
      * @return array
-     *
-     * @throws UnableRetrieveData
+     * @deprecated
+     * @see self::execute
      */
     public function get(array $values) : array
     {
-        $output = [];
+        return $values;
+    }
+
+    /**
+     * Returns data batch.
+     *
+     * @param array $arguments
+     * @param int $batchSize
+     * @return \Generator
+     * @throws \Zend_Db_Statement_Exception
+     */
+    private function getDataBatch(array $arguments, int $batchSize): \Generator
+    {
+        $itemN = 0;
         $queryArguments = [];
         $mappedCategories = [];
         $attributesData = [];
+        foreach ($arguments as $value) {
+            $scope = $value['scopeId'] ?? Store::DEFAULT_STORE_ID;
+            $queryArguments[$scope][$value['categoryId']] = $value['attribute_ids'] ?? [];
+        }
 
-        try {
-            foreach ($values as $value) {
-                $scope = $value['scopeId'] ?? Store::DEFAULT_STORE_ID;
-                $queryArguments[$scope][$value['categoryId']] = $value['attribute_ids'] ?? [];
-            }
-
-            $connection = $this->resourceConnection->getConnection();
-            foreach ($queryArguments as $scopeId => $categoryData) {
-                $cursor = $connection->query(
-                    $this->categoryMainQuery->getQuery(\array_keys($categoryData), $scopeId ?: null)
-                );
+        $connection = $this->resourceConnection->getConnection();
+        foreach ($queryArguments as $scopeId => $categoryData) {
+            $cursor = $connection->query(
+                $this->categoryMainQuery->getQuery(\array_keys($categoryData), $scopeId ?: null)
+            );
 
-                while ($row = $cursor->fetch()) {
-                    $mappedCategories[$row['storeViewCode']][$row['categoryId']] = $row;
-                    $attributesData[$row['storeViewCode']][$row['categoryId']] = $categoryData[$row['categoryId']];
+            while ($row = $cursor->fetch()) {
+                $itemN++;
+                $mappedCategories[$row['storeViewCode']][$row['categoryId']] = $row;
+                $attributesData[$row['storeViewCode']][$row['categoryId']] = $categoryData[$row['categoryId']];
+                if ($itemN % $batchSize == 0) {
+                    yield [$mappedCategories,  $attributesData];
+                    $mappedCategories = [];
+                    $attributesData = [];
                 }
             }
-
-            foreach ($mappedCategories as $storeCode => $categories) {
-                $output[] = \array_map(function ($row) {
-                    return $this->formatter->format($row);
-                }, \array_replace_recursive(
-                    $categories,
-                    $this->entityEavAttributesResolver->resolve($attributesData[$storeCode], $storeCode)
-                ));
-            }
-        } catch (\Throwable $exception) {
-            $this->logger->error($exception->getMessage(), ['exception' => $exception]);
-            throw new UnableRetrieveData('Unable to retrieve category data');
         }
 
-        return !empty($output) ? \array_merge(...$output) : [];
+        yield [$mappedCategories, $attributesData];
     }
 }

CatalogDataExporter/Model/Provider/ProductMetadata.php

@@ -10,13 +10,15 @@
 use Magento\CatalogDataExporter\Model\Provider\Product\Formatter\FormatterInterface;
 use Magento\CatalogDataExporter\Model\Query\ProductMetadataQuery;
 use Magento\DataExporter\Exception\UnableRetrieveData;
+use Magento\DataExporter\Export\DataProcessorInterface;
+use Magento\DataExporter\Model\Indexer\FeedIndexMetadata;
 use Magento\Framework\App\ResourceConnection;
 use Magento\DataExporter\Model\Logging\CommerceDataExportLoggerInterface as LoggerInterface;
 
 /**
  * Product attributes metadata provider
  */
-class ProductMetadata
+class ProductMetadata implements DataProcessorInterface
 {
     /**
      * Category EAV entity type id
@@ -88,24 +90,29 @@ private function format(array $row) : array
 
         // we only retrieve catalog eav attributes (product and category attributes only) in query
         $output['attributeType'] = ((int)$output['entityTypeId'] === self::CATEGORY_EAV_ENTITY_TYPE_ID) ?
-                                        self::CATEGORY_EAV_ENTITY_TYPE : self::PRODUCT_EAV_ENTITY_TYPE;
+            self::CATEGORY_EAV_ENTITY_TYPE : self::PRODUCT_EAV_ENTITY_TYPE;
 
         return $output;
     }
 
     /**
-     * Returns attribute data
+     * @inheritdoc
      *
-     * @param array $values
-     * @return array
      * @throws UnableRetrieveData
+     * @throws \Zend_Db_Statement_Exception
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
-    public function get(array $values): array
-    {
+    public function execute(
+        array $arguments,
+        callable $dataProcessorCallback,
+        FeedIndexMetadata $metadata,
+        $node = null,
+        $info = null
+    ): void {
         $output = [];
         $queryArguments = [];
         try {
-            foreach ($values as $value) {
+            foreach ($arguments as $value) {
                 $queryArguments['id'][$value['id']] = $value['id'];
             }
             $connection = $this->resourceConnection->getConnection();
@@ -118,6 +125,20 @@ public function get(array $values): array
             $this->logger->error($exception->getMessage(), ['exception' => $exception]);
             throw new UnableRetrieveData('Unable to retrieve product data');
         }
-        return $output;
+
+        $dataProcessorCallback($this->get($output));
+    }
+
+    /**
+     * For backward compatibility with existing 3-rd party plugins.
+     *
+     * @param array $values
+     * @return array
+     * @deprecated
+     * @see self::execute
+     */
+    public function get(array $values) : array
+    {
+        return $values;
     }
 }

CatalogDataExporter/Model/Provider/Products.php

@@ -118,7 +118,7 @@ public function execute(
                 $mappedProducts[$row['storeViewCode']][$row['productId']] = $row;
                 $attributesData[$row['storeViewCode']][$row['productId']] = $productData[$row['productId']];
                 if ($itemN % $metadata->getBatchSize() == 0) {
-                    $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback);
+                    $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback, $metadata);
                     $mappedProducts = [];
                     $attributesData = [];
                 }
@@ -129,13 +129,14 @@ public function execute(
             $scopes = \implode(',', \array_unique(\array_column($arguments, 'scopeId')));
             $this->logger->info(
                 \sprintf(
-                    'Product exporter: no product data found for ids %s in scopes %s. Is product deleted or un-assigned from website?',
+                    'Product exporter: no product data found for ids %s in scopes %s.'
+                    . ' Is product deleted or un-assigned from website?',
                     $productsIds,
                     $scopes
                 )
             );
         } else {
-            $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback);
+            $this->processProducts($mappedProducts, $attributesData, $dataProcessorCallback, $metadata);
         }
     }
 
@@ -156,17 +157,22 @@ public function get(array $values) : array
      * @param array $mappedProducts
      * @param array $attributesData
      * @param callable $dataProcessorCallback
+     * @param FeedIndexMetadata $metadata
      * @return void
      * @throws UnableRetrieveData
      */
     private function processProducts(
         array $mappedProducts,
         array $attributesData,
-        callable $dataProcessorCallback
+        callable $dataProcessorCallback,
+        FeedIndexMetadata $metadata
     ): void {
         $output = [];
+        $modifiedAt = (new \DateTime())->format($metadata->getDateTimeFormat());
+
         foreach ($mappedProducts as $storeCode => $products) {
-            $output[] = \array_map(function ($row) {
+            $output[] = \array_map(function ($row) use ($modifiedAt) {
+                $row['modifiedAt'] = $modifiedAt;
                 return $this->formatter->format($row);
             }, \array_replace_recursive(
                 $products,

CatalogDataExporter/Model/Query/ProductMetadataQuery.php

@@ -16,10 +16,11 @@
  */
 class ProductMetadataQuery
 {
+    private const PRODUCT_EAV_ENTITY_TYPE = 'catalog_product';
     /**
      * @var ResourceConnection
      */
-    private $resourceConnection;
+    private ResourceConnection $resourceConnection;
 
     /**
      * @param ResourceConnection $resourceConnection
@@ -43,9 +44,18 @@ public function getQuery(array $arguments): Select
 
         return $connection->select()
             ->from(['eav' => $this->resourceConnection->getTableName('eav_attribute')], [])
+            ->join(
+                ['eav_type' => $this->resourceConnection->getTableName('eav_entity_type')],
+                sprintf(
+                    'eav_type.entity_type_code = "%s" AND eav.entity_type_id = eav_type.entity_type_id',
+                    self::PRODUCT_EAV_ENTITY_TYPE
+                ),
+                []
+            )
             ->join(
                 ['cea' => $this->resourceConnection->getTableName('catalog_eav_attribute')],
-                'eav.attribute_id = cea.attribute_id'
+                'eav.attribute_id = cea.attribute_id',
+                []
             )
             ->joinLeft(
                 ['s' => $this->resourceConnection->getTableName('store')],

CatalogDataExporter/Setup/Patch/Schema/DropLegacyTables.php

@@ -0,0 +1,53 @@
+<?php
+
+/*************************************************************************
+ *
+ * Copyright 2023 Adobe
+ * All Rights Reserved.
+ *
+ * NOTICE: All information contained herein is, and remains
+ * the property of Adobe and its suppliers, if any. The intellectual
+ * and technical concepts contained herein are proprietary to Adobe
+ * and its suppliers and are protected by all applicable intellectual
+ * property laws, including trade secret and copyright laws.
+ * Dissemination of this information or reproduction of this material
+ * is strictly forbidden unless prior written permission is obtained
+ * from Adobe.
+ * ***********************************************************************
+ */
+declare(strict_types=1);
+
+namespace Magento\CatalogDataExporter\Setup\Patch\Schema;
+
+use Magento\DataExporter\Setup\DropLegacyTablesAbstract;
+use Magento\Framework\Setup\Patch\SchemaPatchInterface;
+
+class DropLegacyTables extends DropLegacyTablesAbstract implements SchemaPatchInterface
+{
+    /**
+     * @inheritdoc
+     */
+    public function apply(): self
+    {
+        $tableList = [
+            'catalog_data_exporter_categories',
+            'catalog_data_exporter_product_attributes',
+            'catalog_data_exporter_products'
+        ];
+
+        return $this->dropTables($tableList);
+    }
+
+    public static function getDependencies(): array
+    {
+        return [RenameOldChangeLogTables::class, UpdateHashConfigDataExporterIndex::class];
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function getAliases(): array
+    {
+        return [];
+    }
+}