From c89c4a9a1d23d7cf535f1f20617cd726792ab12d Mon Sep 17 00:00:00 2001
From: Leonid Poluianov <46716220+le0n4ik@users.noreply.github.com>
Date: Wed, 28 Feb 2024 09:46:56 -0600
Subject: [PATCH] MDEE-713: 'is_deleted' usage is hardcoded in MT queries
(#398)
* MDEE-713: 'is_deleted' usage is hardcoded in MT queries
---
CatalogDataExporter/etc/di.xml | 4 +++
DataExporter/Model/Batch/Feed/Generator.php | 3 +-
DataExporter/Model/Batch/Feed/Iterator.php | 21 +++++++++---
.../Model/Batch/FeedChangeLog/Generator.php | 2 +-
.../Model/Indexer/FeedIndexMetadata.php | 2 +-
.../FeedIndexMetadataProviderInterface.php | 32 +++++++++++++++++++
DataExporter/Model/Indexer/FeedIndexer.php | 10 +++++-
.../Model/Indexer/ViewMaterializer.php | 26 +++++++--------
InventoryDataExporter/etc/di.xml | 2 ++
ProductPriceDataExporter/etc/di.xml | 2 ++
ProductReviewDataExporter/etc/di.xml | 4 +++
ProductVariantDataExporter/etc/di.xml | 2 ++
README.md | 5 +--
13 files changed, 91 insertions(+), 24 deletions(-)
create mode 100644 DataExporter/Model/Indexer/FeedIndexMetadataProviderInterface.php
diff --git a/CatalogDataExporter/etc/di.xml b/CatalogDataExporter/etc/di.xml
index 378eb215..42566071 100644
--- a/CatalogDataExporter/etc/di.xml
+++ b/CatalogDataExporter/etc/di.xml
@@ -242,6 +242,8 @@
- websiteCode
+ true
+
- productId
- storeViewCode
@@ -284,6 +286,7 @@
- feed_data
- is_deleted
+ true
@@ -380,6 +383,7 @@
- feed_data
- is_deleted
+ true
diff --git a/DataExporter/Model/Batch/Feed/Generator.php b/DataExporter/Model/Batch/Feed/Generator.php
index 7fb745df..dd5a6c83 100644
--- a/DataExporter/Model/Batch/Feed/Generator.php
+++ b/DataExporter/Model/Batch/Feed/Generator.php
@@ -139,7 +139,8 @@ public function generate(FeedIndexMetadata $metadata, array $args = []): BatchIt
'sourceTableName' => $sourceTableName,
'sourceTableKeyColumns' => $sourceTableKeyColumns,
'batchLocator' => $batchLocator,
- 'dateTimeFormat' => $metadata->getDateTimeFormat()
+ 'dateTimeFormat' => $metadata->getDateTimeFormat(),
+ 'isRemovable' => $metadata->isRemovable()
]
);
}
diff --git a/DataExporter/Model/Batch/Feed/Iterator.php b/DataExporter/Model/Batch/Feed/Iterator.php
index ff639e2b..b2341710 100644
--- a/DataExporter/Model/Batch/Feed/Iterator.php
+++ b/DataExporter/Model/Batch/Feed/Iterator.php
@@ -82,6 +82,13 @@ class Iterator implements BatchIteratorInterface
*/
private ?string $dateTimeFormat;
+ /**
+ * Is removable flag.
+ *
+ * @var bool|null
+ */
+ private ?bool $isRemovable;
+
/**
* @var CommerceDataExportLoggerInterface
*/
@@ -95,6 +102,7 @@ class Iterator implements BatchIteratorInterface
* @param array $sourceTableKeyColumns
* @param CommerceDataExportLoggerInterface $logger
* @param string|null $dateTimeFormat
+ * @param bool|null $isRemovable
*/
public function __construct(
ResourceConnection $resourceConnection,
@@ -103,7 +111,8 @@ public function __construct(
string $sourceTableName,
array $sourceTableKeyColumns,
CommerceDataExportLoggerInterface $logger,
- ?string $dateTimeFormat = null
+ ?string $dateTimeFormat = null,
+ ?bool $isRemovable = null
) {
$this->resourceConnection = $resourceConnection;
$this->batchLocator = $batchLocator;
@@ -112,6 +121,7 @@ public function __construct(
$this->sourceTableKeyColumns = $sourceTableKeyColumns;
$this->dateTimeFormat = $dateTimeFormat;
$this->logger = $logger;
+ $this->isRemovable = $isRemovable;
}
/**
@@ -201,7 +211,6 @@ function ($key) {
['t' => $this->sourceTableName],
[
't.' . FeedIndexMetadata::FEED_TABLE_FIELD_FEED_DATA,
- 't.' . FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED,
't.' . FeedIndexMetadata::FEED_TABLE_FIELD_MODIFIED_AT
]
)
@@ -211,12 +220,16 @@ function ($key) {
[]
)
->where($this->batchTable->getBatchNumberField() . ' = ?', $this->batchNumber);
-
+ if (true === $this->isRemovable) {
+ $select->columns(
+ 't.' . FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED
+ );
+ }
$statement = $connection->query($select);
$data = [];
while ($row = $statement->fetch()) {
$entity = json_decode($row[FeedIndexMetadata::FEED_TABLE_FIELD_FEED_DATA], true);
- $entity['deleted'] = (bool)$row[FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED];
+ $entity['deleted'] = $this->isRemovable && $row[FeedIndexMetadata::FEED_TABLE_FIELD_IS_DELETED];
$entity['modifiedAt'] = $row[FeedIndexMetadata::FEED_TABLE_FIELD_MODIFIED_AT];
if (null !== $this->dateTimeFormat) {
try {
diff --git a/DataExporter/Model/Batch/FeedChangeLog/Generator.php b/DataExporter/Model/Batch/FeedChangeLog/Generator.php
index 6a76f309..dea4d9e3 100644
--- a/DataExporter/Model/Batch/FeedChangeLog/Generator.php
+++ b/DataExporter/Model/Batch/FeedChangeLog/Generator.php
@@ -89,7 +89,7 @@ public function generate(FeedIndexMetadata $metadata, array $args = []): BatchIt
if ($connection instanceof \Magento\ResourceConnections\DB\Adapter\Pdo\MysqlProxy) {
$connection->setUseMasterConnection();
}
- $viewId = $metadata->getFeedTableName();
+ $viewId = $args['viewId'] ?? $metadata->getFeedTableName();
$view = $this->viewFactory->create()->load($viewId);
$sourceTableName = $this->resourceConnection->getTableName($view->getChangelog()->getName());
$sourceTableField = $view->getChangelog()->getColumnName();
diff --git a/DataExporter/Model/Indexer/FeedIndexMetadata.php b/DataExporter/Model/Indexer/FeedIndexMetadata.php
index 9b69f692..eac55a87 100644
--- a/DataExporter/Model/Indexer/FeedIndexMetadata.php
+++ b/DataExporter/Model/Indexer/FeedIndexMetadata.php
@@ -185,7 +185,7 @@ public function __construct(
array $excludeFromHashFields = [],
bool $exportImmediately = false,
bool $persistExportedFeed = false,
- bool $entitiesRemovable = true,
+ bool $entitiesRemovable = false,
?string $dateTimeFormat = null,
?Config $config = null
) {
diff --git a/DataExporter/Model/Indexer/FeedIndexMetadataProviderInterface.php b/DataExporter/Model/Indexer/FeedIndexMetadataProviderInterface.php
new file mode 100644
index 00000000..d1f63aeb
--- /dev/null
+++ b/DataExporter/Model/Indexer/FeedIndexMetadataProviderInterface.php
@@ -0,0 +1,32 @@
+feedIndexMetadata;
+ }
}
diff --git a/DataExporter/Model/Indexer/ViewMaterializer.php b/DataExporter/Model/Indexer/ViewMaterializer.php
index 4e9b9d4f..5d1c1572 100644
--- a/DataExporter/Model/Indexer/ViewMaterializer.php
+++ b/DataExporter/Model/Indexer/ViewMaterializer.php
@@ -22,6 +22,7 @@
use Magento\DataExporter\Model\FeedPool;
use Magento\DataExporter\Model\Logging\CommerceDataExportLoggerInterface;
use Magento\Framework\Mview\ActionFactory;
+use Magento\Framework\Mview\ActionInterface;
use Magento\Framework\Mview\View\ChangelogTableNotExistsException;
use Magento\Framework\Mview\View\StateInterface;
use Magento\Framework\Mview\ViewInterface;
@@ -141,9 +142,9 @@ public function execute(ViewInterface $view): void
private function executeAction(ViewInterface $view)
{
$action = $this->actionFactory->get($view->getActionClass());
- $feedMetadata = $this->getFeedIndexMetadata($view->getId());
+ $feedMetadata = $this->getFeedIndexMetadata($action);
- $batchIterator = $this->batchGenerator->generate($feedMetadata);
+ $batchIterator = $this->batchGenerator->generate($feedMetadata, ['viewId' => $view->getId()]);
$threadCount = min($feedMetadata->getThreadCount(), $batchIterator->count());
$userFunctions = [];
for ($threadNumber = 1; $threadNumber <= $threadCount; $threadNumber++) {
@@ -171,22 +172,19 @@ private function executeAction(ViewInterface $view)
}
/**
- * Returns feed metadata by feed table name.
+ * Returns feed metadata by mview action object
*
- * @param string $feedTableName
+ * @param ActionInterface $action
* @return FeedIndexMetadata
*/
- private function getFeedIndexMetadata(string $feedTableName): FeedIndexMetadata
+ private function getFeedIndexMetadata(ActionInterface $action): FeedIndexMetadata
{
- $feedList = $this->feedPool->getList();
- foreach ($feedList as $feed) {
- if ($feed->getFeedMetadata()->getFeedTableName() === $feedTableName) {
- return $feed->getFeedMetadata();
- }
+ if ($action instanceof FeedIndexMetadataProviderInterface) {
+ return $action->getFeedIndexMetadata();
+ } else {
+ throw new \InvalidArgumentException(
+ \sprintf('Feed for the "%s" action class is not registered', $action::class)
+ );
}
-
- throw new \InvalidArgumentException(
- \sprintf('Feed with "%s" table name is not registered', $feedTableName)
- );
}
}
diff --git a/InventoryDataExporter/etc/di.xml b/InventoryDataExporter/etc/di.xml
index 869f9611..6a484bcd 100644
--- a/InventoryDataExporter/etc/di.xml
+++ b/InventoryDataExporter/etc/di.xml
@@ -33,6 +33,8 @@
- feed_data
- is_deleted
+
+ true
diff --git a/ProductPriceDataExporter/etc/di.xml b/ProductPriceDataExporter/etc/di.xml
index fafcb8a9..ee42d9d5 100644
--- a/ProductPriceDataExporter/etc/di.xml
+++ b/ProductPriceDataExporter/etc/di.xml
@@ -32,6 +32,8 @@
- customerGroupCode
+ true
+
diff --git a/ProductReviewDataExporter/etc/di.xml b/ProductReviewDataExporter/etc/di.xml
index 3b87197d..3784ccbe 100644
--- a/ProductReviewDataExporter/etc/di.xml
+++ b/ProductReviewDataExporter/etc/di.xml
@@ -20,6 +20,8 @@
- feed_data
- is_deleted
+
+ true
@@ -81,6 +83,8 @@
- feed_data
- is_deleted
+
+ true
diff --git a/ProductVariantDataExporter/etc/di.xml b/ProductVariantDataExporter/etc/di.xml
index 65cd98c6..d1cb31d2 100644
--- a/ProductVariantDataExporter/etc/di.xml
+++ b/ProductVariantDataExporter/etc/di.xml
@@ -18,6 +18,8 @@
- feed_data
- is_deleted
+
+ true
diff --git a/README.md b/README.md
index ad642f51..ff3417fb 100644
--- a/README.md
+++ b/README.md
@@ -91,8 +91,9 @@ These codes will be saved in the "status" field of the feed table, to keep infor
### Feed Index Metadata additional parameters:
-- entitiesRemovable - this parameter handles feed configuration to cover cases when feed entities are not removable. Default value: `true` - feed entities can be removed. For example, `sales order` feed exports Sales Orders that cannot be deleted.
-
+- entitiesRemovable - this parameter handles feed configuration to cover cases when feed entities are not removable. Default value: `false` - feed entities can not be removed. For example:
+- `sales order` feed export's Sales Orders entities cannot be deleted and `isRemovable` metadata parameter set to false.
+- `product` feed export's Products can be deleted and `isRemovable` metadata parameter *MUST* be set to true, in other case - feed records wouldn't be marked as deleted in the event of entity removal.
### Multi-thread data export mode:
The purpose of this mode is to speed up the export process by splitting the data into batches and processing them in parallel.
The performance of data export should be aligned with the limit that is defined for a client at consumer side.