diff --git a/app/code/Magento/Analytics/Model/ReportWriter.php b/app/code/Magento/Analytics/Model/ReportWriter.php index c1df4e1af508..f6c18b128df5 100644 --- a/app/code/Magento/Analytics/Model/ReportWriter.php +++ b/app/code/Magento/Analytics/Model/ReportWriter.php @@ -9,6 +9,7 @@ use Magento\Analytics\ReportXml\DB\ReportValidator; use Magento\Framework\Filesystem\Directory\WriteInterface; +use Magento\Framework\Filesystem\File\WriteInterface as FileWriteInterface; /** * Writes reports in files in csv format @@ -69,23 +70,7 @@ public function write(WriteInterface $directory, $path) continue; } } - /** @var $providerObject */ - $providerObject = $this->providerFactory->create($provider['class']); - $fileName = $provider['parameters'] ? $provider['parameters']['name'] : $provider['name']; - $fileFullPath = $path . $fileName . '.csv'; - $fileData = $providerObject->getReport(...array_values($provider['parameters'])); - $stream = $directory->openFile($fileFullPath, 'w+'); - $stream->lock(); - $headers = []; - foreach ($fileData as $row) { - if (!$headers) { - $headers = array_keys($row); - $stream->writeCsv($headers); - } - $stream->writeCsv($this->prepareRow($row)); - } - $stream->unlock(); - $stream->close(); + $this->prepareData($provider, $directory, $path); } if ($errorsList) { $errorStream = $directory->openFile($path . $this->errorsFileName, 'w+'); @@ -100,6 +85,61 @@ public function write(WriteInterface $directory, $path) return true; } + /** + * Prepare report data + * + * @param array $provider + * @param WriteInterface $directory + * @param string $path + * @return void + * @throws \Magento\Framework\Exception\FileSystemException + */ + private function prepareData(array $provider, WriteInterface $directory, string $path) + { + /** @var $providerObject */ + $providerObject = $this->providerFactory->create($provider['class']); + $fileName = $provider['parameters'] ? $provider['parameters']['name'] : $provider['name']; + $fileFullPath = $path . $fileName . '.csv'; + + $stream = $directory->openFile($fileFullPath, 'w+'); + $stream->lock(); + + $headers = []; + if ($providerObject instanceof \Magento\Analytics\ReportXml\BatchReportProviderInterface) { + $fileData = $providerObject->getBatchReport(...array_values($provider['parameters'])); + do { + $this->doWrite($fileData, $stream, $headers); + $fileData = $providerObject->getBatchReport(...array_values($provider['parameters'])); + $fileData->rewind(); + } while ($fileData->valid()); + } else { + $fileData = $providerObject->getReport(...array_values($provider['parameters'])); + $this->doWrite($fileData, $stream, $headers); + } + + $stream->unlock(); + $stream->close(); + } + + /** + * Write data to file + * + * @param \Traversable $fileData + * @param FileWriteInterface $stream + * @param array $headers + * @return void + */ + private function doWrite(\Traversable $fileData, FileWriteInterface $stream, array $headers) + { + foreach ($fileData as $row) { + if (!$headers) { + $headers = array_keys($row); + $stream->writeCsv($headers); + } + $stream->writeCsv($this->prepareRow($row)); + } + } + /** * Replace wrong symbols in row * diff --git a/app/code/Magento/Analytics/ReportXml/BatchReportProviderInterface.php b/app/code/Magento/Analytics/ReportXml/BatchReportProviderInterface.php new file mode 100644 index 000000000000..350956a7886c --- /dev/null +++ b/app/code/Magento/Analytics/ReportXml/BatchReportProviderInterface.php @@ -0,0 +1,32 @@ + $this->getConfig() ]; } + + /** + * Get SQL for get record count + * + * @return Select + * @throws \Zend_Db_Select_Exception + */ + public function getSelectCountSql(): Select + { + if (!$this->selectCount) { + $this->selectCount = clone $this->getSelect(); + $this->selectCount->reset(\Magento\Framework\DB\Select::ORDER); + $this->selectCount->reset(\Magento\Framework\DB\Select::LIMIT_COUNT); + $this->selectCount->reset(\Magento\Framework\DB\Select::LIMIT_OFFSET); + $this->selectCount->reset(\Magento\Framework\DB\Select::COLUMNS); + + $part = $this->getSelect()->getPart(\Magento\Framework\DB\Select::GROUP); + if (!is_array($part) || !count($part)) { + $this->selectCount->columns(new \Zend_Db_Expr('COUNT(*)')); + return $this->selectCount; + } + + $this->selectCount->reset(\Magento\Framework\DB\Select::GROUP); + $group = $this->getSelect()->getPart(\Magento\Framework\DB\Select::GROUP); + $this->selectCount->columns(new \Zend_Db_Expr(("COUNT(DISTINCT ".implode(", ", $group).")"))); + } + return $this->selectCount; + } } diff --git a/app/code/Magento/Analytics/ReportXml/ReportProvider.php b/app/code/Magento/Analytics/ReportXml/ReportProvider.php index 8966d018dc6b..05b2f8b44653 100644 --- a/app/code/Magento/Analytics/ReportXml/ReportProvider.php +++ b/app/code/Magento/Analytics/ReportXml/ReportProvider.php @@ -11,7 +11,7 @@ /** * Providers for reports data */ -class ReportProvider +class ReportProvider implements BatchReportProviderInterface { /** * @var QueryFactory @@ -28,6 +28,26 @@ class ReportProvider */ private $iteratorFactory; + /** + * @var int + */ + private $currentPosition = 0; + + /** + * @var int + */ + private $countTotal = 0; + + /** + * @var \Magento\Framework\DB\Adapter\AdapterInterface + */ + private $connection; + + /** + * @var Query + */ + private $dataSelect; + /** * ReportProvider constructor. * @@ -46,8 +66,7 @@ public function __construct( } /** - * Returns custom iterator name for report - * Null for default + * Returns custom iterator name for report. Null for default * * @param Query $query * @return string|null @@ -71,4 +90,27 @@ public function getReport($name) $statement = $connection->query($query->getSelect()); return $this->iteratorFactory->create($statement, $this->getIteratorName($query)); } + + /** + * @inheritdoc + */ + public function getBatchReport(string $name): \IteratorIterator + { + if (!$this->dataSelect || $this->dataSelect->getConfig()['name'] !== $name) { + $this->dataSelect = $this->queryFactory->create($name); + $this->currentPosition = 0; + $this->connection = $this->connectionFactory->getConnection($this->dataSelect->getConnectionName()); + $this->countTotal = $this->connection->fetchOne($this->dataSelect->getSelectCountSql()); + } + + if ($this->currentPosition >= $this->countTotal) { + return $this->iteratorFactory->create(new \ArrayIterator([]), $this->getIteratorName($this->dataSelect)); + } + + $statement = $this->connection->query( + $this->dataSelect->getSelect()->limit(self::BATCH_SIZE, $this->currentPosition) + ); + $this->currentPosition += self::BATCH_SIZE; + return $this->iteratorFactory->create($statement, $this->getIteratorName($this->dataSelect)); + } } diff --git a/app/code/Magento/Analytics/Test/Unit/Model/ReportWriterTest.php b/app/code/Magento/Analytics/Test/Unit/Model/ReportWriterTest.php index e718a4c84c94..cc8b9d1bca01 100644 --- a/app/code/Magento/Analytics/Test/Unit/Model/ReportWriterTest.php +++ b/app/code/Magento/Analytics/Test/Unit/Model/ReportWriterTest.php @@ -107,6 +107,8 @@ protected function setUp(): void */ public function testWrite(array $configData, array $fileData, array $expectedFileData): void { + $fileData = new \IteratorIterator(new \ArrayIterator($fileData)); + $emptyFileData = new \IteratorIterator(new \ArrayIterator([])); $errors = []; $this->configInterfaceMock ->expects($this->once()) @@ -121,10 +123,10 @@ public function testWrite(array $configData, array $fileData, array $expectedFil $parameterName = isset(reset($configData)[0]['parameters']['name']) ? reset($configData)[0]['parameters']['name'] : ''; - $this->reportProviderMock->expects($this->once()) - ->method('getReport') + $this->reportProviderMock->expects($this->exactly(2)) + ->method('getBatchReport') ->with($parameterName ?: null) - ->willReturn($fileData); + ->willReturnOnConsecutiveCalls($fileData, $emptyFileData); $errorStreamMock = $this->getMockBuilder( FileWriteInterface::class )->getMockForAbstractClass(); diff --git a/app/code/Magento/Analytics/Test/Unit/ReportXml/QueryTest.php b/app/code/Magento/Analytics/Test/Unit/ReportXml/QueryTest.php index d1a2afc4697c..c00c88c4e6ef 100644 --- a/app/code/Magento/Analytics/Test/Unit/ReportXml/QueryTest.php +++ b/app/code/Magento/Analytics/Test/Unit/ReportXml/QueryTest.php @@ -84,4 +84,31 @@ public function testJsonSerialize() $this->assertSame($expectedResult, $this->query->jsonSerialize()); } + + public function testGetSelectCountSql() + { + $resetParams = [ + Select::ORDER, + Select::LIMIT_COUNT, + Select::LIMIT_OFFSET, + Select::COLUMNS + ]; + + $this->selectMock + ->expects($this->exactly(4)) + ->method('reset') + ->willReturnCallback( + function (string $value) use (&$resetParams) { + $this->assertEquals(array_shift($resetParams), $value); + } + ); + + $this->selectMock + ->expects($this->once()) + ->method('columns') + ->with(new \Zend_Db_Expr('COUNT(*)')) + ->willReturnSelf(); + + $this->assertEquals($this->selectMock, $this->query->getSelectCountSql()); + } } diff --git a/app/code/Magento/Analytics/Test/Unit/ReportXml/ReportProviderTest.php b/app/code/Magento/Analytics/Test/Unit/ReportXml/ReportProviderTest.php index 98b291c580be..02a36df29dbc 100644 --- a/app/code/Magento/Analytics/Test/Unit/ReportXml/ReportProviderTest.php +++ b/app/code/Magento/Analytics/Test/Unit/ReportXml/ReportProviderTest.php @@ -155,4 +155,55 @@ public function testGetReport() ->willReturn($this->iteratorMock); $this->assertEquals($this->iteratorMock, $this->subject->getReport($reportName)); } + + /** + * @return void + */ + public function testGetBatchReport() + { + $reportName = 'test_report'; + $connectionName = 'sales'; + + $this->queryFactoryMock->expects($this->once()) + ->method('create') + ->with($reportName) + ->willReturn($this->queryMock); + + $this->connectionFactoryMock->expects($this->once()) + ->method('getConnection') + ->with($connectionName) + ->willReturn($this->connectionMock); + + $this->queryMock->expects($this->once()) + ->method('getConnectionName') + ->willReturn($connectionName); + + $this->selectMock->expects($this->once()) + ->method('limit') + ->with(ReportProvider::BATCH_SIZE, 0) + ->willReturn($this->selectMock); + + $this->queryMock->expects($this->once()) + ->method('getConfig') + ->willReturn( + [ + 'connection' => $connectionName + ] + ); + + $this->connectionMock->expects($this->once()) + ->method('query') + ->with($this->selectMock) + ->willReturn($this->statementMock); + + $this->connectionMock->expects($this->once()) + ->method('fetchOne') + ->willReturn(5); + + $this->iteratorFactoryMock->expects($this->once()) + ->method('create') + ->with($this->statementMock, null) + ->willReturn($this->iteratorMock); + $this->assertEquals($this->iteratorMock, $this->subject->getBatchReport($reportName)); + } } diff --git a/app/code/Magento/Catalog/Block/Widget/RecentlyViewed.php b/app/code/Magento/Catalog/Block/Widget/RecentlyViewed.php index ee78e7345637..29ac5945664f 100644 --- a/app/code/Magento/Catalog/Block/Widget/RecentlyViewed.php +++ b/app/code/Magento/Catalog/Block/Widget/RecentlyViewed.php @@ -13,4 +13,5 @@ */ class RecentlyViewed extends Wrapper implements \Magento\Widget\Block\BlockInterface { + protected const RENDER_TYPE = 'html'; } diff --git a/app/code/Magento/Ui/Block/Wrapper.php b/app/code/Magento/Ui/Block/Wrapper.php index bab9fd7bb785..497bb4d28584 100644 --- a/app/code/Magento/Ui/Block/Wrapper.php +++ b/app/code/Magento/Ui/Block/Wrapper.php @@ -14,6 +14,8 @@ */ class Wrapper extends \Magento\Framework\View\Element\Template { + protected const RENDER_TYPE = ''; + /** * @var UiComponentGenerator */ @@ -91,6 +93,6 @@ public function renderApp($data = []) ->generateUiComponent($this->getData('uiComponent'), $this->getLayout()); $this->injectDataInDataSource($uiComponent, $this->getData()); $this->addDataToChildComponents($uiComponent, $data); - return (string) $uiComponent->render(); + return (string) $uiComponent->render(static::RENDER_TYPE); } } diff --git a/app/code/Magento/Ui/Component/Listing.php b/app/code/Magento/Ui/Component/Listing.php index 513186560ca0..d8b372e9dfe9 100644 --- a/app/code/Magento/Ui/Component/Listing.php +++ b/app/code/Magento/Ui/Component/Listing.php @@ -5,6 +5,9 @@ */ namespace Magento\Ui\Component; +use Magento\Framework\App\ObjectManager; +use Magento\Framework\View\Element\UiComponent\ContentType\ContentTypeFactory; +use Magento\Framework\View\Element\UiComponent\ContextInterface; use Magento\Ui\Component\Listing\Columns; /** @@ -13,13 +16,34 @@ */ class Listing extends AbstractComponent { - const NAME = 'listing'; + public const NAME = 'listing'; /** * @var array */ protected $columns = []; + /** + * @var ContentTypeFactory + */ + private ContentTypeFactory $contentTypeFactory; + + /** + * @param ContextInterface $context + * @param array $components + * @param array $data + * @param ContentTypeFactory|null $contentTypeFactory + */ + public function __construct( + ContextInterface $context, + array $components = [], + array $data = [], + ?ContentTypeFactory $contentTypeFactory = null + ) { + $this->contentTypeFactory = $contentTypeFactory ?: ObjectManager::getInstance()->get(ContentTypeFactory::class); + parent::__construct($context, $components, $data); + } + /** * Get component name * @@ -31,10 +55,25 @@ public function getComponentName() } /** - * {@inheritdoc} + * @inheritdoc */ public function getDataSourceData() { return ['data' => $this->getContext()->getDataProvider()->getData()]; } + + /** + * Render content depending on specified type + * + * @param string $contentType + * @return string + */ + public function render(string $contentType = '') + { + if ($contentType) { + return $this->contentTypeFactory->get($contentType)->render($this, $this->getTemplate()); + } + + return parent::render(); + } } diff --git a/app/code/Magento/Ui/Test/Unit/Component/ListingTest.php b/app/code/Magento/Ui/Test/Unit/Component/ListingTest.php index ae5d4bf4138d..bf0812e4351e 100644 --- a/app/code/Magento/Ui/Test/Unit/Component/ListingTest.php +++ b/app/code/Magento/Ui/Test/Unit/Component/ListingTest.php @@ -8,6 +8,9 @@ namespace Magento\Ui\Test\Unit\Component; use Magento\Framework\TestFramework\Unit\Helper\ObjectManager; +use Magento\Framework\View\Element\UiComponent\ContentType\AbstractContentType; +use Magento\Framework\View\Element\UiComponent\ContentType\ContentTypeFactory; +use Magento\Framework\View\Element\UiComponent\ContentType\Html; use Magento\Framework\View\Element\UiComponent\ContextInterface; use Magento\Framework\View\Element\UiComponent\Processor; use Magento\Ui\Component\Listing; @@ -26,6 +29,11 @@ class ListingTest extends TestCase */ protected $objectManager; + /** + * @var ContentTypeFactory|MockObject + */ + private ContentTypeFactory $contentTypeFactory; + /** * @inheritdoc */ @@ -39,6 +47,8 @@ protected function setUp(): void '', false ); + + $this->contentTypeFactory = $this->createMock(ContentTypeFactory::class); } /** @@ -103,4 +113,48 @@ public function testPrepare(): void $listing->prepare(); } + + /** + * @return void + */ + public function testRenderSpecificContentType(): void + { + $html = 'html output'; + $renderer = $this->createMock(Html::class); + $renderer->expects($this->once())->method('render')->willReturn($html); + $this->contentTypeFactory->expects($this->once()) + ->method('get') + ->with('html') + ->willReturn($renderer); + + /** @var Listing $listing */ + $listing = $this->objectManager->getObject( + Listing::class, + [ + 'context' => $this->contextMock, + 'contentTypeFactory' => $this->contentTypeFactory + ] + ); + $this->assertSame($html, $listing->render('html')); + } + + /** + * @return void + */ + public function testRenderParent() + { + $html = 'html output'; + $renderer = $this->createMock(AbstractContentType::class); + $renderer->expects($this->once())->method('render')->willReturn($html); + $this->contextMock->expects($this->once())->method('getRenderEngine')->willReturn($renderer); + + /** @var Listing $listing */ + $listing = $this->objectManager->getObject( + Listing::class, + [ + 'context' => $this->contextMock + ] + ); + $this->assertSame($html, $listing->render()); + } } diff --git a/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php b/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php index fc84dcafc468..500795a25d65 100644 --- a/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php +++ b/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php @@ -1812,13 +1812,8 @@ private function prepareColumnData(array $ddl): array } } - /** - * Starting from MariaDB 10.5.1 columns with old temporal formats are marked with a \/* mariadb-5.3 *\/ - * comment in the output of SHOW CREATE TABLE, SHOW COLUMNS, DESCRIBE statements, - * as well as in the COLUMN_TYPE column of the INFORMATION_SCHEMA.COLUMNS Table. - */ foreach ($ddl as $key => $columnData) { - $ddl[$key]['DATA_TYPE'] = str_replace(' /* mariadb-5.3 */', '', $columnData['DATA_TYPE']); + $ddl[$key]['DATA_TYPE'] = $this->sanitizeColumnDataType($columnData['DATA_TYPE']); } return $ddl; @@ -1975,7 +1970,7 @@ public function modifyColumnByDdl($tableName, $columnName, $definition, $flushDa protected function _getColumnTypeByDdl($column) { // phpstan:ignore - switch ($column['DATA_TYPE']) { + switch ($this->sanitizeColumnDataType($column['DATA_TYPE'])) { case 'bool': return Table::TYPE_BOOLEAN; case 'tinytext': @@ -2012,6 +2007,22 @@ protected function _getColumnTypeByDdl($column) return null; } + /** + * Remove old temporal format comment from column data type + * + * @param string $columnType + * @return string + */ + private function sanitizeColumnDataType(string $columnType): string + { + /** + * Starting from MariaDB 10.5.1 columns with old temporal formats are marked with a \/* mariadb-5.3 *\/ + * comment in the output of SHOW CREATE TABLE, SHOW COLUMNS, DESCRIBE statements, + * as well as in the COLUMN_TYPE column of the INFORMATION_SCHEMA.COLUMNS Table. + */ + return str_replace(' /* mariadb-5.3 */', '', $columnType); + } + /** * Change table storage engine * @@ -2584,6 +2595,8 @@ protected function _getColumnDefinition($options, $ddlType = null) // detect and validate column type if ($ddlType === null) { $ddlType = $this->_getDdlType($options); + } else { + $ddlType = $this->sanitizeColumnDataType($ddlType); } if (empty($ddlType) || !isset($this->_ddlColumnTypes[$ddlType])) { @@ -3225,6 +3238,8 @@ public function prepareColumnValue(array $column, $value) return $value; } + $column['DATA_TYPE'] = $this->sanitizeColumnDataType($column['DATA_TYPE']); + // return original value if invalid column describe data if (!isset($column['DATA_TYPE'])) { return $value; @@ -3994,7 +4009,7 @@ protected function _getDdlType($options) $ddlType = $options['COLUMN_TYPE']; } - return $ddlType; + return $this->sanitizeColumnDataType($ddlType); } /** diff --git a/lib/internal/Magento/Framework/DB/Test/Unit/Adapter/Pdo/MysqlTest.php b/lib/internal/Magento/Framework/DB/Test/Unit/Adapter/Pdo/MysqlTest.php index 92a704ff8428..ec58f10361ea 100644 --- a/lib/internal/Magento/Framework/DB/Test/Unit/Adapter/Pdo/MysqlTest.php +++ b/lib/internal/Magento/Framework/DB/Test/Unit/Adapter/Pdo/MysqlTest.php @@ -852,6 +852,129 @@ public static function columnDataForTest(): array ]; } + /** + * @param array $actual + * @param int|string|\Zend_Db_Expr $expected + * @dataProvider columnDataAndValueForTest + * @return void + * @throws \ReflectionException + */ + public function testPrepareColumnValue(array $actual, int|string|\Zend_Db_Expr $expected) + { + $adapter = $this->getMysqlPdoAdapterMock([]); + + $result = $this->invokeModelMethod($adapter, 'prepareColumnValue', [$actual[0], $actual[1]]); + + $this->assertEquals($expected, $result); + } + + /** + * Data provider for testPrepareColumnValue + * + * @return array[] + */ + public function columnDataAndValueForTest(): array + { + return [ + [ + 'actual' => [ + [ + 'DATA_TYPE' => 'int', + 'DEFAULT' => '' + ], + '10' + ], + 'expected' => 10 + ], + [ + 'actual' => [ + [ + 'DATA_TYPE' => 'datetime /* mariadb-5.3 */', + 'DEFAULT' => 'CURRENT_TIMESTAMP' + ], + 'null' + ], + 'expected' => new \Zend_Db_Expr('NULL') + ], + [ + 'actual' => [ + [ + 'DATA_TYPE' => 'date /* mariadb-5.3 */', + 'DEFAULT' => '' + ], + 'null' + ], + 'expected' => new \Zend_Db_Expr('NULL') + ], + [ + 'actual' => [ + [ + 'DATA_TYPE' => 'timestamp /* mariadb-5.3 */', + 'DEFAULT' => 'CURRENT_TIMESTAMP' + ], + 'null' + ], + 'expected' => new \Zend_Db_Expr('NULL') + ], + [ + 'actual' => [ + [ + 'DATA_TYPE' => 'varchar', + 'NULLABLE' => false, + 'DEFAULT' => '' + ], + 10 + ], + 'expected' => '10' + ] + ]; + } + + /** + * @param string $actual + * @param string $expected + * @dataProvider providerForSanitizeColumnDataType + * @return void + * @throws \ReflectionException + */ + public function testSanitizeColumnDataType(string $actual, string $expected) + { + $adapter = $this->getMysqlPdoAdapterMock([]); + $result = $this->invokeModelMethod($adapter, 'sanitizeColumnDataType', [$actual]); + $this->assertEquals($expected, $result); + } + + /** + * Data provider for testSanitizeColumnDataType + * + * @return array[] + */ + public function providerForSanitizeColumnDataType() + { + return [ + [ + 'actual' => 'int', + 'expected' => 'int' + ], + [ + 'actual' => 'varchar', + 'expected' => 'varchar' + ], + [ + 'actual' => 'datetime /* mariadb-5.3 */', + 'expected' => 'datetime' + ], + [ + 'actual' => 'date /* mariadb-5.3 */', + 'expected' => 'date' + ], + [ + 'actual' => 'timestamp /* mariadb-5.3 */', + 'expected' => 'timestamp' + ] + ]; + } + /** * @param string $method * @param array $parameters