Merge branch 'main' of https://github.com/mage-os/mirror-commerce-dat…

…a-export

CatalogDataExporter/Test/Integration/ProductBatchTest.php

@@ -0,0 +1,65 @@
+<?php
+/*************************************************************************
+ *
+ * Copyright 2023 Adobe
+ * All Rights Reserved.
+ *
+ * NOTICE: All information contained herein is, and remains
+ * the property of Adobe and its suppliers, if any. The intellectual
+ * and technical concepts contained herein are proprietary to Adobe
+ * and its suppliers and are protected by all applicable intellectual
+ * property laws, including trade secret and copyright laws.
+ * Dissemination of this information or reproduction of this material
+ * is strictly forbidden unless prior written permission is obtained
+ * from Adobe.
+ * ***********************************************************************
+ */
+declare(strict_types=1);
+
+namespace Magento\CatalogDataExporter\Test\Integration;
+
+use Magento\Catalog\Test\Fixture\Product as ProductFixture;
+use Magento\DataExporter\Model\Batch\FeedSource\Generator;
+use Magento\DataExporter\Model\FeedInterface;
+use Magento\DataExporter\Model\FeedPool;
+use Magento\TestFramework\Fixture\Config;
+use Magento\TestFramework\Fixture\DataFixture;
+use Magento\TestFramework\Fixture\DbIsolation;
+use Magento\TestFramework\Helper\Bootstrap;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Test feed source batch generation for products.
+ */
+class ProductBatchTest extends TestCase
+{
+    /**
+     * @var ?Generator
+     */
+    private ?Generator $productFeedSourceGenerator;
+    public ?FeedInterface $productFeed;
+
+    /**
+     * Integration test setup
+     */
+    protected function setUp(): void
+    {
+        parent::setUp();
+        $this->productFeed = Bootstrap::getObjectManager()->get(FeedPool::class)->getFeed('products');
+        $this->productFeedSourceGenerator = Bootstrap::getObjectManager()->create(Generator::class);
+    }
+
+    #[
+        Config('commerce_data_export/feeds/products/batch_size', 5),
+        DbIsolation(false),
+        DataFixture(ProductFixture::class, count: 10),
+    ]
+    public function testProductFeedSourceGenerator() : void
+    {
+        $batchIterator = $this->productFeedSourceGenerator->generate($this->productFeed->getFeedMetadata());
+        self::assertEquals(2, $batchIterator->count(), 'Batch count is wrong');
+        foreach ($batchIterator as $ids) {
+            self::assertCount(5, $ids, 'The number of products in batch is wrong');
+        }
+    }
+}

CatalogDataExporter/etc/config.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!--
+ /************************************************************************
+  *
+  * Copyright 2023 Adobe
+  * All Rights Reserved.
+  *
+  * NOTICE: All information contained herein is, and remains
+  * the property of Adobe and its suppliers, if any. The intellectual
+  * and technical concepts contained herein are proprietary to Adobe
+  * and its suppliers and are protected by all applicable intellectual
+  * property laws, including trade secret and copyright laws.
+  * Dissemination of this information or reproduction of this material
+  * is strictly forbidden unless prior written permission is obtained
+  * from Adobe.
+  * **********************************************************************
+  */
+-->
+<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:module:Magento_Store:etc/config.xsd">
+    <default>
+        <commerce_data_export>
+            <feeds>
+                <products>
+                    <thread_count>1</thread_count>
+                    <batch_size>100</batch_size>
+                </products>
+            </feeds>
+        </commerce_data_export>
+    </default>
+</config>

CatalogDataExporter/etc/di.xml

@@ -497,10 +497,6 @@
         </arguments>
     </type>
 
-    <type name="Magento\Framework\Mview\Processor">
-        <plugin name="exporter-cover-exception" type="Magento\CatalogDataExporter\Plugin\CoverExceptionMview"/>
-    </type>
-
     <!-- handle product attribute deletion  -->
     <type name="Magento\Catalog\Model\ResourceModel\Eav\Attribute">
         <plugin name="exporter-reindex-on-product-attribute-delete" type="Magento\CatalogDataExporter\Plugin\Eav\Attribute\ProductAttributeDelete"/>

DataExporter/Model/Batch/BatchGeneratorInterface.php

@@ -0,0 +1,33 @@
+<?php
+/************************************************************************
+ *
+ * Copyright 2023 Adobe
+ * All Rights Reserved.
+ *
+ * NOTICE: All information contained herein is, and remains
+ * the property of Adobe and its suppliers, if any. The intellectual
+ * and technical concepts contained herein are proprietary to Adobe
+ * and its suppliers and are protected by all applicable intellectual
+ * property laws, including trade secret and copyright laws.
+ * Dissemination of this information or reproduction of this material
+ * is strictly forbidden unless prior written permission is obtained
+ * from Adobe.
+ * ************************************************************************
+ */
+declare(strict_types=1);
+
+namespace Magento\DataExporter\Model\Batch;
+
+use Magento\DataExporter\Model\Indexer\FeedIndexMetadata;
+
+interface BatchGeneratorInterface
+{
+    /**
+     * Creates data batches based on feed index metadata.
+     *
+     * @param FeedIndexMetadata $metadata
+     * @param array $args
+     * @return BatchIteratorInterface
+     */
+    public function generate(FeedIndexMetadata $metadata, array $args = []): BatchIteratorInterface;
+}

DataExporter/Model/Batch/BatchIteratorInterface.php

@@ -0,0 +1,29 @@
+<?php
+/************************************************************************
+ *
+ * Copyright 2023 Adobe
+ * All Rights Reserved.
+ *
+ * NOTICE: All information contained herein is, and remains
+ * the property of Adobe and its suppliers, if any. The intellectual
+ * and technical concepts contained herein are proprietary to Adobe
+ * and its suppliers and are protected by all applicable intellectual
+ * property laws, including trade secret and copyright laws.
+ * Dissemination of this information or reproduction of this material
+ * is strictly forbidden unless prior written permission is obtained
+ * from Adobe.
+ * ************************************************************************
+ */
+declare(strict_types=1);
+
+namespace Magento\DataExporter\Model\Batch;
+
+interface BatchIteratorInterface extends \Iterator, \Countable
+{
+    /**
+     * Mark batch items for retry.
+     *
+     * @return void
+     */
+    public function markBatchForRetry(): void;
+}

DataExporter/Model/Batch/BatchLocator.php

@@ -0,0 +1,115 @@
+<?php
+/*************************************************************************
+ *
+ * Copyright 2023 Adobe
+ * All Rights Reserved.
+ *
+ * NOTICE: All information contained herein is, and remains
+ * the property of Adobe and its suppliers, if any. The intellectual
+ * and technical concepts contained herein are proprietary to Adobe
+ * and its suppliers and are protected by all applicable intellectual
+ * property laws, including trade secret and copyright laws.
+ * Dissemination of this information or reproduction of this material
+ * is strictly forbidden unless prior written permission is obtained
+ * from Adobe.
+ * ***********************************************************************
+ */
+declare(strict_types=1);
+
+namespace Magento\DataExporter\Model\Batch;
+
+use Magento\Framework\App\ResourceConnection;
+
+/**
+ * Locates the current batch number.
+ */
+class BatchLocator
+{
+    /**
+     * @var ResourceConnection
+     */
+    private ResourceConnection $resourceConnection;
+
+    /**
+     * @var string
+     */
+    private string $sequenceTableName;
+
+    /**
+     * @var int
+     */
+    private int $autoIncrement = 1;
+
+    /**
+     * @var int
+     */
+    private int $autoIncrementOffset = 1;
+
+    /**
+     * @param ResourceConnection $resourceConnection
+     * @param string $sequenceTableName
+     */
+    public function __construct(
+        ResourceConnection $resourceConnection,
+        string $sequenceTableName
+    ) {
+        $this->resourceConnection = $resourceConnection;
+        $this->sequenceTableName = $sequenceTableName;
+    }
+
+    /**
+     * Initializes the batch locator.
+     *
+     * @return void
+     * @throws \Zend_Db_Exception
+     */
+    public function init(): void
+    {
+        $connection = $this->resourceConnection->getConnection();
+        $mutexTable = $connection->newTable($this->sequenceTableName)
+            ->addColumn(
+                'i',
+                \Magento\Framework\DB\Ddl\Table::TYPE_INTEGER,
+                10,
+                [
+                    'identity' => true,
+                    'unsigned' => true,
+                    'nullable' => false,
+                    'primary' => true,
+                ],
+                'Auto Increment ID'
+            );
+        $connection->dropTable($this->sequenceTableName);
+        $connection->createTable($mutexTable);
+        // phpcs:ignore Magento2.SQL.RawQuery.FoundRawSql
+        $sql = 'SELECT @@auto_increment_increment as auto_increment, @@auto_increment_offset as auto_increment_offset';
+        $result = $connection->query($sql)->fetch();
+        $this->autoIncrement = (int)$result['auto_increment'];
+        $this->autoIncrementOffset = (int)$result['auto_increment_offset'];
+    }
+
+    /**
+     * Returns batch number.
+     *
+     * @return int
+     */
+    public function getNumber(): int
+    {
+        $connection = $this->resourceConnection->getConnection();
+        $connection->insert($this->sequenceTableName, []);
+        $lastInsertId = (int)$connection->lastInsertId($this->sequenceTableName);
+        $batchNumber = ($lastInsertId - $this->autoIncrementOffset) / $this->autoIncrement + 1;
+
+        return (int)$batchNumber;
+    }
+
+    /**
+     * Destroys the batch locator.
+     *
+     * @return void
+     */
+    public function destroy(): void
+    {
+        $this->resourceConnection->getConnection()->dropTable($this->sequenceTableName);
+    }
+}