Merge pull request #783 from matomo-org/4-14-2-release

Update Matomo core to 4.13.2

CHANGELOG.md

@@ -1,4 +1,7 @@
 == Changelog ===
+= 4.14.2 =
+* Update Matomo core to 4.14.2
+
 = 4.14.1 =
 * Update Matomo core to 4.14.1
 

app/core/Archive.php

@@ -321,6 +321,20 @@ public function getBlob($names, $idSubtable = null)
         return $this->get($names, 'blob', $idSubtable);
     }
 
+    /**
+     * Queries blob data for a single record. Uses a cursor instead of fetching all the data at once,
+     * and makes sure the result set's order allows aggregating the data one row at a time.
+     *
+     * @param string $name The record name to fetch.
+     * @return \Generator
+     * @internal
+     */
+    public function querySingleBlob($name)
+    {
+        $archiveIds = $this->getArchiveIds([$name]);
+        return ArchiveSelector::querySingleBlob($archiveIds, $name);
+    }
+
     /**
      * Queries and returns metric data in a DataTable instance.
      *
@@ -582,7 +596,7 @@ protected function get($archiveNames, $archiveDataType, $idSubtable = null)
      * query archive tables for IDs w/o launching archiving, or launch archiving and
      * get the idarchive from ArchiveProcessor instances.
      *
-     * @param string $archiveNames
+     * @param string[] $archiveNames
      * @return array
      */
     private function getArchiveIds($archiveNames)

app/core/Archive/DataCollection.php

@@ -381,35 +381,6 @@ private function putRowInIndex(&$index, $metadataNamesToIndexBy, $row, $idSite,
         $currentLevel = $row;
     }
 
-    public function forEachBlobExpanded($callable, $idSubTable = null, $depth = null, $addMetadataSubTableId = false)
-    {
-        $this->checkExpandedMethodPrerequisites();
-
-        $dataTableFactory = new DataTableFactory(
-            $this->dataNames, 'blob', $this->sitesId, $this->periods, $this->segment, $this->defaultRow);
-        $dataTableFactory->expandDataTable($depth, $addMetadataSubTableId);
-        $dataTableFactory->useSubtable($idSubTable);
-
-        foreach ($this->data as $idSite => $periods) {
-            foreach ($periods as $periodRange => $data) {
-                // FIXME: This hack works around a strange bug that occurs when getting
-                //         archive IDs through ArchiveProcessing instances. When a table
-                //         does not already exist, for some reason the archive ID for
-                //         today (or from two days ago) will be added to the Archive
-                //         instances list. The Archive instance will then select data
-                //         for periods outside of the requested set.
-                //         working around the bug here, but ideally, we need to figure
-                //         out why incorrect idarchives are being selected.
-                if (empty($this->periods[$periodRange])) {
-                    continue;
-                }
-                $tableMetadata = $dataTableFactory->getTableMetadataFor($idSite, $this->periods[$periodRange]);
-
-                $callable($data, $dataTableFactory, $tableMetadata);
-            }
-        }
-    }
-
     private function checkExpandedMethodPrerequisites()
     {
         if ($this->dataType != 'blob') {

app/core/ArchiveProcessor.php

@@ -9,7 +9,6 @@
 namespace Piwik;
 
 use Exception;
-use Piwik\Archive\DataCollection;
 use Piwik\Archive\DataTableFactory;
 use Piwik\ArchiveProcessor\Parameters;
 use Piwik\ArchiveProcessor\Rules;
@@ -178,7 +177,9 @@ public function getLogAggregator()
      * @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`.
      * @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable.
      * @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable.
-     * @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
+     * @param string|null $defaultColumnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
+     *                                                           If not set, and the table contains nb_visits or INDEX_NB_VISITS, we will
+     *                                                           sort by visits.
      * @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}.
      * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
      *                                               when summed because they cannot be summed, eg,
@@ -200,7 +201,7 @@ public function getLogAggregator()
     public function aggregateDataTableRecords($recordNames,
                                               $maximumRowsInDataTableLevelZero = null,
                                               $maximumRowsInSubDataTable = null,
-                                              $columnToSortByBeforeTruncation = null,
+                                              $defaultColumnToSortByBeforeTruncation = null,
                                               &$columnsAggregationOperation = null,
                                               $columnsToRenameAfterAggregation = null,
                                               $countRowsRecursive = true)
@@ -230,6 +231,16 @@ public function aggregateDataTableRecords($recordNames,
                 $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
             }
 
+            $columnToSortByBeforeTruncation = $defaultColumnToSortByBeforeTruncation;
+            if (empty($columnToSortByBeforeTruncation)) {
+                $columns = $table->getColumns();
+                if (in_array(Metrics::INDEX_NB_VISITS, $columns)) {
+                    $columnToSortByBeforeTruncation = Metrics::INDEX_NB_VISITS;
+                } else if (in_array('nb_visits', $columns)) {
+                    $columnToSortByBeforeTruncation = 'nb_visits';
+                }
+            }
+
             $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
             Common::destroy($table);
             $this->insertBlobRecord($recordName, $blob);
@@ -355,33 +366,36 @@ protected function aggregateDataTableRecord($name, $columnsAggregationOperation
         try {
             ErrorHandler::pushFatalErrorBreadcrumb(__CLASS__, ['name' => $name]);
 
-            // By default we shall aggregate all sub-tables.
-            $dataTableBlobs = $this->getArchive()->getBlob($name, Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES);
-            $dataTable = $this->getAggregatedDataTableMapFromBlobs($dataTableBlobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name);
+            $blobs = $this->getArchive()->querySingleBlob($name);
+            $dataTable = $this->getAggregatedDataTableMapFromBlobs($blobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name);
         } finally {
             ErrorHandler::popFatalErrorBreadcrumb();
         }
 
         return $dataTable;
     }
 
-    protected function getAggregatedDataTableMapFromBlobs(DataCollection $dataTableBlobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name)
+    protected function getAggregatedDataTableMapFromBlobs(\Iterator $dataTableBlobs, $columnsAggregationOperation, $columnsToRenameAfterAggregation, $name)
     {
+        // maps period & subtable ID in database to the Row instance in $result that subtable should be added to when encountered
+        // [$row['date1'].','.$row['date2']][$tableId] = $row in $result
+        /** @var Row[][] */
+        $tableIdToResultRowMapping = [];
+
         $result = new DataTable();
 
         if (!empty($columnsAggregationOperation)) {
             $result->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
         }
 
-        $dataTableBlobs->forEachBlobExpanded(function ($reportBlobs, DataTableFactory $factory, $tableMetadata) use ($name, $result, $columnsToRenameAfterAggregation) {
-            $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
-
-            $toSum = $factory->make($reportBlobs, $index = [], $tableMetadata);
+        foreach ($dataTableBlobs as $archiveDataRow) {
+            $period = $archiveDataRow['date1'] . ',' . $archiveDataRow['date2'];
+            $tableId = $archiveDataRow['name'] == $name ? null : $this->getSubtableIdFromBlobName($archiveDataRow['name']);
 
-            $latestUsedAfterCreatingToSum = Manager::getInstance()->getMostRecentTableId();
+            $blobTable = DataTable::fromSerializedArray($archiveDataRow['value']);
 
             // see https://github.com/piwik/piwik/issues/4377
-            $toSum->filter(function ($table) use ($columnsToRenameAfterAggregation, $name) {
+            $blobTable->filter(function ($table) use ($columnsToRenameAfterAggregation, $name) {
                 if ($this->areColumnsNotAlreadyRenamed($table)) {
                     /**
                      * This makes archiving and range dates a lot faster. Imagine we archive a week, then we will
@@ -395,14 +409,64 @@ protected function getAggregatedDataTableMapFromBlobs(DataCollection $dataTableB
                 }
             });
 
-            $result->addDataTable($toSum);
+            $tableToAddTo = null;
+            if ($tableId === null) {
+                $tableToAddTo = $result;
+            } else if (empty($tableIdToResultRowMapping[$period][$tableId])) { // sanity check
+                StaticContainer::get(LoggerInterface::class)->info(
+                    'Unexpected state when aggregating DataTable, unknown period/table ID combination encountered: {period} - {tableId}.'
+                    . ' This either means the SQL to order blobs is behaving incorrectly or the blob data is corrupt in some way.',
+                    [
+                        'period' => $period,
+                        'tableId' => $tableId,
+                    ]
+                );
+                continue;
+            } else {
+                $rowToAddTo = $tableIdToResultRowMapping[$period][$tableId];
+
+                if (!$rowToAddTo->getIdSubDataTable()) {
+                    $newTable = new DataTable();
+                    $newTable->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
+                    $rowToAddTo->setSubtable($newTable);
+                }
 
-            DataTable\Manager::getInstance()->deleteAll($latestUsedTableId, $latestUsedAfterCreatingToSum);
-        });
+                $tableToAddTo = $rowToAddTo->getSubtable();
+            }
+
+            $tableToAddTo->addDataTable($blobTable);
+
+            // add subtable IDs for $blobTableRow to $tableIdToResultRowMapping
+            foreach ($blobTable->getRows() as $blobTableRow) {
+                $label = $blobTableRow->getColumn('label');
+                $subtableId = $blobTableRow->getIdSubDataTable();
+                if (empty($subtableId)) {
+                    continue;
+                }
+
+                $rowToAddTo = $tableToAddTo->getRowFromLabel($label);
+                $tableIdToResultRowMapping[$period][$subtableId] = $rowToAddTo;
+            }
+
+            Common::destroy($blobTable);
+            unset($blobTable);
+        }
 
         return $result;
     }
 
+    private function getSubtableIdFromBlobName($recordName)
+    {
+        $parts = explode('_', $recordName);
+        $id = end($parts);
+
+        if (is_numeric($id)) {
+            return $id;
+        }
+
+        return null;
+    }
+
     /**
      * Note: public only for use in closure in PHP 5.3.
      *