Skip to content

Commit

Permalink
[Bug](materialized-view) SelectMaterializedIndexWithAggregate do not … (
Browse files Browse the repository at this point in the history 
apache#26192)

SelectMaterializedIndexWithAggregate do not change plan when match base index
  • Loading branch information
@BiteTheDDDDt
BiteTheDDDDt authored Nov 2, 2023
1 parent 3ce1bda commit 3004dbb
Show file tree
Hide file tree
Showing 8 changed files with 89 additions and 46 deletions.
3 changes: 3 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,9 @@ private void analyzeFromClause() throws AnalysisException {
if (tableRefList.size() != 1) {
throw new AnalysisException("The materialized view only support one table in from clause.");
}
if (!isReplay && tableRefList.get(0).hasExplicitAlias()) {
throw new AnalysisException("The materialized view not support table with alias.");
}
TableName tableName = tableRefList.get(0).getName();
if (tableName == null) {
throw new AnalysisException("table in from clause is invalid, please check if it's single table "
Expand Down
40 changes: 40 additions & 0 deletions .../java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ public List<Rule> buildRules() {
agg.getGroupByExpressions(),
new HashSet<>(agg.getExpressions()));

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);

Expand Down Expand Up @@ -162,6 +166,10 @@ public List<Rule> buildRules() {
requiredExpr
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -207,6 +215,10 @@ public List<Rule> buildRules() {
collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -265,6 +277,10 @@ public List<Rule> buildRules() {
requiredExpr
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -322,6 +338,10 @@ public List<Rule> buildRules() {
requiredExpr
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -369,6 +389,10 @@ public List<Rule> buildRules() {
nonVirtualGroupByExprs(agg),
new HashSet<>(agg.getExpressions()));

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);

Expand Down Expand Up @@ -422,6 +446,10 @@ public List<Rule> buildRules() {
requiredExpr
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -474,6 +502,10 @@ public List<Rule> buildRules() {
collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -539,6 +571,10 @@ public List<Rule> buildRules() {
requiredExpr
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down Expand Up @@ -605,6 +641,10 @@ public List<Rule> buildRules() {
requiredExpr
);

if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}

LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
Expand Down
13 changes: 4 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,12 +178,8 @@ public OlapTable getTable() {

@Override
public String toString() {
return Utils.toSqlString("LogicalOlapScan",
"qualified", qualifiedName(),
"indexName", getSelectedMaterializedIndexName().orElse("<index_not_selected>"),
"selectedIndexId", selectedIndexId,
"preAgg", preAggStatus
);
return Utils.toSqlString("LogicalOlapScan", "qualified", qualifiedName(), "indexName",
getSelectedMaterializedIndexName(), "selectedIndexId", selectedIndexId, "preAgg", preAggStatus);
}

@Override
Expand Down Expand Up @@ -291,9 +287,8 @@ public PreAggStatus getPreAggStatus() {
}

@VisibleForTesting
public Optional<String> getSelectedMaterializedIndexName() {
return indexSelected ? Optional.ofNullable(((OlapTable) table).getIndexNameById(selectedIndexId))
: Optional.empty();
public String getSelectedMaterializedIndexName() {
return ((OlapTable) table).getIndexNameById(selectedIndexId);
}

@Override
Expand Down
17 changes: 5 additions & 12 deletions fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -875,18 +875,11 @@ void testTwoTupleInQuery() throws Exception {
String query = "select * from (select user_id, bitmap_union_count(to_bitmap(tag_id)) x from "
+ USER_TAG_TABLE_NAME + " group by user_id) a, (select user_name, bitmap_union_count(to_bitmap(tag_id))"
+ "" + " y from " + USER_TAG_TABLE_NAME + " group by user_name) b where a.x=b.y;";
PlanChecker.from(connectContext)
.analyze(query)
.rewrite()
.matches(logicalJoin(
logicalProject(
logicalAggregate(
logicalOlapScan().when(scan -> "user_tags_mv".equals(
scan.getSelectedMaterializedIndexName().get())))),
logicalAggregate(
logicalProject(
logicalOlapScan().when(scan -> "user_tags".equals(
scan.getSelectedMaterializedIndexName().get()))))));
PlanChecker.from(connectContext).analyze(query).rewrite().matches(logicalJoin(
logicalProject(logicalAggregate(logicalOlapScan()
.when(scan -> "user_tags_mv".equals(scan.getSelectedMaterializedIndexName())))),
logicalAggregate(logicalProject(logicalOlapScan()
.when(scan -> "user_tags".equals(scan.getSelectedMaterializedIndexName()))))));

}

Expand Down
31 changes: 10 additions & 21 deletions ...e-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectRollupIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public void testMatchingBase() {
.applyTopDown(new SelectMaterializedIndexWithAggregate())
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getPreAggStatus().isOn());
Assertions.assertEquals("t", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("t", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand All @@ -124,7 +124,7 @@ void testAggFilterScan() {
.applyTopDown(new SelectMaterializedIndexWithAggregate())
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getPreAggStatus().isOn());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand All @@ -139,9 +139,6 @@ void testTranslate() {
public void testTranslateWhenPreAggIsOff() {
singleTableTest("select k2, min(v1) from t group by k2", scan -> {
Assertions.assertFalse(scan.isPreAggregation());
Assertions.assertEquals("Aggregate operator don't match, "
+ "aggregate function: min(v1), column aggregate type: SUM",
scan.getReasonOfPreAggregation());
});
}

Expand All @@ -152,7 +149,7 @@ public void testWithEqualFilter() {
.applyTopDown(new SelectMaterializedIndexWithAggregate())
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getPreAggStatus().isOn());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand All @@ -164,7 +161,7 @@ public void testWithNonEqualFilter() {
.applyTopDown(new SelectMaterializedIndexWithAggregate())
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getPreAggStatus().isOn());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand All @@ -176,7 +173,7 @@ public void testWithFilter() {
.applyTopDown(new SelectMaterializedIndexWithAggregate())
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getPreAggStatus().isOn());
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand All @@ -195,7 +192,7 @@ public void testWithFilterAndProject() {
.applyTopDown(new SelectMaterializedIndexWithoutAggregate())
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getPreAggStatus().isOn());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand Down Expand Up @@ -227,8 +224,6 @@ public void testAggregateTypeNotMatch() {
.matches(logicalOlapScan().when(scan -> {
PreAggStatus preAgg = scan.getPreAggStatus();
Assertions.assertTrue(preAgg.isOff());
Assertions.assertEquals("Aggregate operator don't match, "
+ "aggregate function: min(v1), column aggregate type: SUM", preAgg.getOffReason());
return true;
}));
}
Expand All @@ -242,8 +237,6 @@ public void testInvalidSlotInAggFunction() {
.matches(logicalOlapScan().when(scan -> {
PreAggStatus preAgg = scan.getPreAggStatus();
Assertions.assertTrue(preAgg.isOff());
Assertions.assertEquals("Slot((v1 + 1)) in sum((v1 + 1)) is neither key column nor value column.",
preAgg.getOffReason());
return true;
}));
}
Expand All @@ -257,8 +250,6 @@ public void testKeyColumnInAggFunction() {
.matches(logicalOlapScan().when(scan -> {
PreAggStatus preAgg = scan.getPreAggStatus();
Assertions.assertTrue(preAgg.isOff());
Assertions.assertEquals("Aggregate function sum(k2) contains key column k2.",
preAgg.getOffReason());
return true;
}));
}
Expand All @@ -273,7 +264,7 @@ public void testMaxCanUseKeyColumn() {
.matches(logicalOlapScan().when(scan -> {
PreAggStatus preAgg = scan.getPreAggStatus();
Assertions.assertTrue(preAgg.isOn());
Assertions.assertEquals("r4", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r4", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand All @@ -288,7 +279,7 @@ public void testMinCanUseKeyColumn() {
.matches(logicalOlapScan().when(scan -> {
PreAggStatus preAgg = scan.getPreAggStatus();
Assertions.assertTrue(preAgg.isOn());
Assertions.assertEquals("r4", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r4", scan.getSelectedMaterializedIndexName());
return true;
}));
}
Expand Down Expand Up @@ -376,8 +367,6 @@ public void testCountDistinctKeyColumn() {
public void testCountDistinctValueColumn() {
singleTableTest("select k1, count(distinct v1) from t group by k1", scan -> {
Assertions.assertFalse(scan.isPreAggregation());
Assertions.assertEquals("Count distinct is only valid for key columns, but meet count(DISTINCT v1).",
scan.getReasonOfPreAggregation());
Assertions.assertEquals("t", scan.getSelectedIndexName());
});
}
Expand Down Expand Up @@ -425,7 +414,7 @@ public void testPreAggHint() throws Exception {
.rewrite()
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getHints().isEmpty());
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName());
PreAggStatus preAggStatus = scan.getPreAggStatus();
Assertions.assertTrue(preAggStatus.isOff());
Assertions.assertEquals("No aggregate on scan.", preAggStatus.getOffReason());
Expand All @@ -444,7 +433,7 @@ public void testPreAggHint() throws Exception {
.matches(logicalOlapScan().when(scan -> {
Assertions.assertEquals(1, scan.getHints().size());
Assertions.assertEquals("PREAGGOPEN", scan.getHints().get(0));
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get());
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName());
PreAggStatus preAggStatus = scan.getPreAggStatus();
Assertions.assertTrue(preAggStatus.isOn());
return true;
Expand Down
6 changes: 2 additions & 4 deletions fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,8 @@ public void testLogicalJoin(@Mocked Plan left, @Mocked Plan right) {
@Test
public void testLogicalOlapScan() {
LogicalOlapScan plan = PlanConstructor.newLogicalOlapScan(0, "table", 0);
Assertions.assertTrue(
plan.toString().matches("LogicalOlapScan \\( qualified=db\\.table, "
+ "indexName=<index_not_selected>, "
+ "selectedIndexId=-1, preAgg=ON \\)"));
Assertions.assertTrue(plan.toString().matches("LogicalOlapScan \\( qualified=db\\.table, " + "indexName=table, "
+ "selectedIndexId=-1, preAgg=ON \\)"), plan.toString());
}

@Test
Expand Down
8 changes: 8 additions & 0 deletions regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,11 @@
1 1
2 2

-- !select_mv --
1 2
2 2

-- !select_mv --
1 2
2 2

17 changes: 17 additions & 0 deletions regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ suite ("testProjectionMV1") {
sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""

test {
sql "create materialized view emps_mv as select deptno, empid from emps t order by deptno;"
exception "errCode = 2,"
}

createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;")

sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
Expand All @@ -50,4 +55,16 @@ suite ("testProjectionMV1") {
contains "(emps_mv)"
}
qt_select_mv "select empid, deptno from emps order by empid;"

explain {
sql("select empid, sum(deptno) from emps group by empid order by empid;")
contains "(emps_mv)"
}
qt_select_mv "select empid, sum(deptno) from emps group by empid order by empid;"

explain {
sql("select deptno, sum(empid) from emps group by deptno order by deptno;")
contains "(emps_mv)"
}
qt_select_mv "select deptno, sum(empid) from emps group by deptno order by deptno;"
}

0 comments on commit 3004dbb

Please sign in to comment.