Skip to content

Commit

Permalink
add logic plan builder
Browse files Browse the repository at this point in the history 
fix null pointer

add process logic

modify refresh manager logic

modify refresh manager logic
  • Loading branch information
@vinlee19
vinlee19 authored and vinlee19 committed Nov 7, 2024
1 parent b17397f commit 741b486
Show file tree
Hide file tree
Showing 6 changed files with 238 additions and 66 deletions.
8 changes: 6 additions & 2 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ statementBase
| supportedDropStatement #supportedDropStatementAlias
| supportedSetStatement #supportedSetStatementAlias
| supportedUnsetStatement #supportedUnsetStatementAlias
| supportedRefreshStatement #supportedRefreshStatementAlias
| supportedRefreshStatement #supportedRefreshStatementAlias
| unsupportedStatement #unsupported
;

Expand All @@ -75,6 +75,7 @@ unsupportedStatement
| unsupportedCancelStatement
| unsupportedJobStatement
| unsupportedCleanStatement
| unsupportedRefreshStatement
| unsupportedLoadStatement
| unsupportedShowStatement
| unsupportedOtherStatement
Expand Down Expand Up @@ -409,9 +410,12 @@ channelDescription
;

supportedRefreshStatement
: REFRESH CATALOG name=identifier propertyClause? #refreshCatalog
;

unsupportedRefreshStatement
: REFRESH TABLE name=multipartIdentifier #refreshTable
| REFRESH DATABASE name=multipartIdentifier propertyClause? #refreshDatabase
| REFRESH CATALOG name=identifier propertyClause? #refreshCatalog
| REFRESH LDAP (ALL | (FOR user=identifierOrText)) #refreshLdap
;

Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.persist.OperationType;
import org.apache.doris.qe.DdlExecutor;

import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -284,9 +284,9 @@ public void run() {
* {@link org.apache.doris.analysis.RefreshCatalogStmt#analyze(Analyzer)} is ok,
* because the default value of invalidCache is true.
* */
RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt(catalogName, null);
RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand(catalogName, null);
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
refreshCatalogCommand.handleRefreshCatalog();
} catch (Exception e) {
LOG.warn("failed to refresh catalog {}", catalogName, e);
}
Expand Down
118 changes: 65 additions & 53 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@
import org.apache.doris.nereids.DorisParser.QueryContext;
import org.apache.doris.nereids.DorisParser.QueryOrganizationContext;
import org.apache.doris.nereids.DorisParser.QueryTermContext;
import org.apache.doris.nereids.DorisParser.RefreshCatalogContext;
import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
Expand Down Expand Up @@ -475,6 +476,7 @@
import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
Expand Down Expand Up @@ -1209,7 +1211,7 @@ public LogicalPlan visitLoad(DorisParser.LoadContext ctx) {
List<String> tableName = RelationUtil.getQualifierName(ConnectContext.get(), nameParts);
List<String> colNames = (ddc.columns == null ? ImmutableList.of() : visitIdentifierList(ddc.columns));
List<String> columnsFromPath = (ddc.columnsFromPath == null ? ImmutableList.of()
: visitIdentifierList(ddc.columnsFromPath.identifierList()));
: visitIdentifierList(ddc.columnsFromPath.identifierList()));
List<String> partitions = ddc.partition == null ? ImmutableList.of() : visitIdentifierList(ddc.partition);
// TODO: multi location
List<String> multiFilePaths = new ArrayList<>();
Expand All @@ -1228,16 +1230,16 @@ public LogicalPlan visitLoad(DorisParser.LoadContext ctx) {
}

LoadTask.MergeType mergeType = ddc.mergeType() == null ? LoadTask.MergeType.APPEND
: LoadTask.MergeType.valueOf(ddc.mergeType().getText());
: LoadTask.MergeType.valueOf(ddc.mergeType().getText());

Optional<String> fileFormat = ddc.format == null ? Optional.empty()
: Optional.of(visitIdentifierOrText(ddc.format));
Optional<String> separator = ddc.separator == null ? Optional.empty() : Optional.of(ddc.separator.getText()
.substring(1, ddc.separator.getText().length() - 1));
.substring(1, ddc.separator.getText().length() - 1));
Optional<String> comma = ddc.comma == null ? Optional.empty() : Optional.of(ddc.comma.getText()
.substring(1, ddc.comma.getText().length() - 1));
.substring(1, ddc.comma.getText().length() - 1));
Map<String, String> dataProperties = ddc.propertyClause() == null ? new HashMap<>()
: visitPropertyClause(ddc.propertyClause());
: visitPropertyClause(ddc.propertyClause());
dataDescriptions.add(new BulkLoadDataDesc(
tableName,
partitions,
Expand Down Expand Up @@ -1313,8 +1315,8 @@ public LogicalSubQueryAlias<Plan> visitAliasQuery(AliasQueryContext ctx) {
LogicalPlan queryPlan = plan(ctx.query());
Optional<List<String>> columnNames = optionalVisit(ctx.columnAliases(), () ->
ctx.columnAliases().identifier().stream()
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList())
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList())
);
return new LogicalSubQueryAlias<>(ctx.identifier().getText(), columnNames, queryPlan);
});
Expand Down Expand Up @@ -1733,7 +1735,7 @@ public Expression visitComparison(ComparisonContext ctx) {
return new NullSafeEqual(left, right);
default:
throw new ParseException("Unsupported comparison expression: "
+ operator.getSymbol().getText(), ctx);
+ operator.getSymbol().getText(), ctx);
}
});
}
Expand Down Expand Up @@ -1978,7 +1980,7 @@ public Expression visitDate_add(Date_addContext ctx) {
Expression timeStamp = (Expression) visit(ctx.timestamp);
Expression amount = (Expression) visit(ctx.unitsAmount);
if (ctx.unit == null) {
//use "DAY" as unit by default
// use "DAY" as unit by default
return new DaysAdd(timeStamp, amount);
}

Expand Down Expand Up @@ -2040,7 +2042,7 @@ public Expression visitDate_sub(Date_subContext ctx) {
Expression timeStamp = (Expression) visit(ctx.timestamp);
Expression amount = (Expression) visit(ctx.unitsAmount);
if (ctx.unit == null) {
//use "DAY" as unit by default
// use "DAY" as unit by default
return new DaysSub(timeStamp, amount);
}

Expand Down Expand Up @@ -2172,11 +2174,11 @@ public Expression visitDoublePipes(DorisParser.DoublePipesContext ctx) {
/**
* Create a value based [[CaseWhen]] expression. This has the following SQL form:
* {{{
* CASE [expression]
* WHEN [value] THEN [expression]
* ...
* ELSE [expression]
* END
* CASE [expression]
* WHEN [value] THEN [expression]
* ...
* ELSE [expression]
* END
* }}}
*/
@Override
Expand All @@ -2194,11 +2196,11 @@ public Expression visitSimpleCase(DorisParser.SimpleCaseContext context) {
/**
* Create a condition based [[CaseWhen]] expression. This has the following SQL syntax:
* {{{
* CASE
* WHEN [predicate] THEN [expression]
* ...
* ELSE [expression]
* END
* CASE
* WHEN [predicate] THEN [expression]
* ...
* ELSE [expression]
* END
* }}}
*
* @param context the parse tree
Expand Down Expand Up @@ -2349,8 +2351,8 @@ private WindowExpression withWindowSpec(WindowSpecContext ctx, Expression functi
List<OrderExpression> orderKeyList = Lists.newArrayList();
if (ctx.sortClause() != null) {
orderKeyList = visit(ctx.sortClause().sortItem(), OrderKey.class).stream()
.map(orderKey -> new OrderExpression(orderKey))
.collect(Collectors.toList());
.map(orderKey -> new OrderExpression(orderKey))
.collect(Collectors.toList());
}

if (ctx.windowFrame() != null) {
Expand Down Expand Up @@ -2641,8 +2643,8 @@ public LogicalPlan visitRelationList(DorisParser.RelationListContext ctx) {
@Override
public List<String> visitMultipartIdentifier(MultipartIdentifierContext ctx) {
return ctx.parts.stream()
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList());
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList());
}

/**
Expand All @@ -2659,8 +2661,8 @@ public List<String> visitIdentifierList(IdentifierListContext ctx) {
@Override
public List<String> visitIdentifierSeq(IdentifierSeqContext ctx) {
return ctx.ident.stream()
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList());
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList());
}

@Override
Expand Down Expand Up @@ -2841,10 +2843,10 @@ public PartitionTableInfo visitPartitionTable(DorisParser.PartitionTableContext
})
.collect(ImmutableList.toImmutableList());
return new PartitionTableInfo(
isAutoPartition,
ctx.RANGE() != null ? "RANGE" : "LIST",
ctx.partitions != null ? visitPartitionsDef(ctx.partitions) : null,
partitionList);
isAutoPartition,
ctx.RANGE() != null ? "RANGE" : "LIST",
ctx.partitions != null ? visitPartitionsDef(ctx.partitions) : null,
partitionList);
}

@Override
Expand Down Expand Up @@ -2916,7 +2918,7 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) {
e.getCause());
}
}
//comment should remove '\' and '(") at the beginning and end
// comment should remove '\' and '(") at the beginning and end
String comment = ctx.comment != null ? ctx.comment.getText().substring(1, ctx.comment.getText().length() - 1)
.replace("\\", "") : "";
long autoIncInitValue = -1;
Expand Down Expand Up @@ -2950,9 +2952,9 @@ public IndexDefinition visitIndexDef(IndexDefContext ctx) {
List<String> indexCols = visitIdentifierList(ctx.cols);
Map<String, String> properties = visitPropertyItemList(ctx.properties);
String indexType = ctx.indexType != null ? ctx.indexType.getText().toUpperCase() : null;
//comment should remove '\' and '(") at the beginning and end
// comment should remove '\' and '(") at the beginning and end
String comment = ctx.comment == null ? "" : LogicalPlanBuilderAssistant.escapeBackSlash(
ctx.comment.getText().substring(1, ctx.STRING_LITERAL().getText().length() - 1));
ctx.comment.getText().substring(1, ctx.STRING_LITERAL().getText().length() - 1));
// change BITMAP index to INVERTED index
if (Config.enable_create_bitmap_index_as_inverted_index
&& "BITMAP".equalsIgnoreCase(indexType)) {
Expand Down Expand Up @@ -3401,7 +3403,7 @@ public Object visitCommentRelationHint(CommentRelationHintContext ctx) {
}

protected LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx,
Optional<AggClauseContext> aggCtx, boolean isDistinct) {
Optional<AggClauseContext> aggCtx, boolean isDistinct) {
return ParserUtils.withOrigin(selectCtx, () -> {
if (aggCtx.isPresent()) {
if (isDistinct) {
Expand Down Expand Up @@ -3446,12 +3448,12 @@ private LogicalPlan withRelations(LogicalPlan inputPlan, List<RelationContext> r

private LogicalPlan withFilter(LogicalPlan input, Optional<WhereClauseContext> whereCtx) {
return input.optionalMap(whereCtx, () ->
new LogicalFilter<>(ExpressionUtils.extractConjunctionToSet(
getExpression(whereCtx.get().booleanExpression())), input));
new LogicalFilter<>(ExpressionUtils.extractConjunctionToSet(
getExpression(whereCtx.get().booleanExpression())), input));
}

private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext selectCtx,
Optional<AggClauseContext> aggCtx) {
Optional<AggClauseContext> aggCtx) {
return input.optionalMap(aggCtx, () -> {
GroupingElementContext groupingElementContext = aggCtx.get().groupingElement();
List<NamedExpression> namedExpressions = getNamedExpressions(selectCtx.namedExpressionSeq());
Expand Down Expand Up @@ -3505,15 +3507,15 @@ private Expression withPredicate(Expression valueExpression, PredicateContext ct
break;
case DorisParser.LIKE:
outExpression = new Like(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.RLIKE:
case DorisParser.REGEXP:
outExpression = new Regexp(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.IN:
Expand Down Expand Up @@ -3544,38 +3546,38 @@ private Expression withPredicate(Expression valueExpression, PredicateContext ct
case DorisParser.MATCH:
case DorisParser.MATCH_ANY:
outExpression = new MatchAny(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.MATCH_ALL:
outExpression = new MatchAll(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.MATCH_PHRASE:
outExpression = new MatchPhrase(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.MATCH_PHRASE_PREFIX:
outExpression = new MatchPhrasePrefix(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.MATCH_REGEXP:
outExpression = new MatchRegexp(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
case DorisParser.MATCH_PHRASE_EDGE:
outExpression = new MatchPhraseEdge(
valueExpression,
getExpression(ctx.pattern)
valueExpression,
getExpression(ctx.pattern)
);
break;
default:
Expand Down Expand Up @@ -3999,4 +4001,14 @@ public SetUserPropertiesCommand visitSetUserProperties(SetUserPropertiesContext
public SetDefaultStorageVaultCommand visitSetDefaultStorageVault(SetDefaultStorageVaultContext ctx) {
return new SetDefaultStorageVaultCommand(stripQuotes(ctx.identifier().getText()));
}

@Override
public Object visitRefreshCatalog(RefreshCatalogContext ctx) {

if (ctx.name != null) {
String catalogName = ctx.name.getText();
return new RefreshCatalogCommand(catalogName, new HashMap<>());
}
throw new AnalysisException("catalog name can not be null");
}
}
Loading

0 comments on commit 741b486

Please sign in to comment.