Skip to content

Commit

Permalink
[BugFix] Disable date_trunc equivalent replace if binary type is LE (…
Browse files Browse the repository at this point in the history 
…backport #53229) (#53276)

Signed-off-by: shuming.li <[email protected]>
Co-authored-by: shuming.li <[email protected]>
  • Loading branch information
@mergify @LiShuMing
mergify[bot] and LiShuMing authored Nov 29, 2024
1 parent aca82f5 commit b455786
Show file tree
Hide file tree
Showing 4 changed files with 240 additions and 25 deletions.
3 changes: 2 additions & 1 deletion ...a/com/starrocks/sql/optimizer/rule/transformation/materialization/PredicateExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ private static RangePredicate rewriteByEquivalent(BinaryPredicateOperator predic
}

TreeRangeSet<ConstantOperator> range = range(predicate.getBinaryType(), op2);
if (DateTruncEquivalent.INSTANCE.isEquivalent(op1, op2)) {
if (DateTruncEquivalent.isSupportedBinaryType(predicate.getBinaryType()) &&
DateTruncEquivalent.INSTANCE.isEquivalent(op1, op2)) {
TreeRangeSet<ConstantOperator> rangeSet = TreeRangeSet.create();
rangeSet.addAll(range);
return new ColumnRangePredicate(op1.getChild(1).cast(), rangeSet);
Expand Down
54 changes: 42 additions & 12 deletions ...cks/sql/optimizer/rule/transformation/materialization/equivalent/DateTruncEquivalent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
// limitations under the License.
package com.starrocks.sql.optimizer.rule.transformation.materialization.equivalent;

import com.google.common.collect.ImmutableSet;
import com.starrocks.analysis.BinaryType;
import com.starrocks.catalog.FunctionSet;
import com.starrocks.catalog.PrimitiveType;
import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
Expand All @@ -22,11 +24,35 @@
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
import com.starrocks.sql.optimizer.rewrite.ScalarOperatorFunctions;

import java.util.Set;

public class DateTruncEquivalent extends IPredicateRewriteEquivalent {
public static final DateTruncEquivalent INSTANCE = new DateTruncEquivalent();

public DateTruncEquivalent() {}

/**
* TODO: we can support this later.
* Change date_trunc('month', dt) to col = '2023-12-01' will get a wrong result.
* MV : select date_trunc('day', dt) as dt from t
* Query1 : select date_trunc('month, dt) from t dt = '2023-11-01'
* -- cannot be rewritten, rewrite result will be wrong
* Rewritten: select date_trunc('month, dt) from t where date_trunc('month', dt) = '2023-11-01'
*
* Query2 : select date_trunc('month, dt) from t where dt between '2023-11-01' and '2023-12-01'
* -- cannot be rewritten, dt='2023-12-01' doesn't match with date_trunc('month', dt)= '2023-11-01'
* Rewritten : select date_trunc('month, dt) from t where date_trunc('month', dt) between '2023-11-01' and '2023-12-01'
*/
private static Set<BinaryType> SUPPORTED_BINARY_TYPES = ImmutableSet.of(
BinaryType.GE,
BinaryType.GT,
BinaryType.LT
);

public static boolean isSupportedBinaryType(BinaryType binaryType) {
return SUPPORTED_BINARY_TYPES.contains(binaryType);
}

@Override
public boolean isEquivalent(ScalarOperator op1, ConstantOperator op2) {
if (!(op1 instanceof CallOperator)) {
Expand Down Expand Up @@ -68,20 +94,24 @@ public ScalarOperator rewrite(RewriteEquivalentContext eqContext,
EquivalentShuttleContext shuttleContext,
ColumnRefOperator replace,
ScalarOperator newInput) {
if (!(newInput instanceof BinaryPredicateOperator)) {
return null;
}
ScalarOperator left = newInput.getChild(0);
ScalarOperator right = newInput.getChild(1);
if (newInput instanceof BinaryPredicateOperator) {
ScalarOperator left = newInput.getChild(0);
ScalarOperator right = newInput.getChild(1);

if (!right.isConstantRef() || !left.equals(eqContext.getEquivalent())) {
return null;
}
if (!isEquivalent(eqContext.getInput(), (ConstantOperator) right)) {
if (!right.isConstantRef() || !left.equals(eqContext.getEquivalent())) {
return null;
}
if (!isEquivalent(eqContext.getInput(), (ConstantOperator) right)) {
return null;
}
BinaryPredicateOperator predicate = (BinaryPredicateOperator) newInput.clone();
if (!isSupportedBinaryType(predicate.getBinaryType())) {
return null;
}
predicate.setChild(0, replace);
return predicate;
} else {
return null;
}
BinaryPredicateOperator predicate = (BinaryPredicateOperator) newInput.clone();
predicate.setChild(0, replace);
return predicate;
}
}
200 changes: 192 additions & 8 deletions fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManualTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,26 +250,91 @@ public void testDateTruncPartitionColumnExpr() throws Exception {
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"COMMENT \"MATERIALIZED_VIEW\"\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY HASH(`order_num`) BUCKETS 10\n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"date_trunc('minute', `dt`) AS ds,\n" +
"bitmap_union(to_bitmap(`order_id`)) AS `order_num` \n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('minute', `dt`) BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.match("test_partition_expr_mv1");
}
{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}
starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
starRocksAssert.dropTable("test_partition_expr_tbl1");
}

@Test
public void testDateTruncPartitionColumnExpr2() throws Exception {
String tableSQL = "CREATE TABLE `test_partition_expr_tbl1` (\n" +
" `order_id` bigint(20) NOT NULL DEFAULT \"-1\" COMMENT \"\",\n" +
" `dt` datetime NOT NULL DEFAULT \"1996-01-01 00:00:00\" COMMENT \"\",\n" +
" `value` varchar(256) NULL \n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`order_id`, `dt`)\n" +
"PARTITION BY RANGE(`dt`)\n" +
"(\n" +
"PARTITION p2023041017 VALUES [(\"2023-04-10 17:00:00\"), (\"2023-04-10 18:00:00\")),\n" +
"PARTITION p2023041021 VALUES [(\"2023-04-10 21:00:00\"), (\"2023-04-10 22:00:00\"))\n" +
")\n" +
"DISTRIBUTED BY HASH(`order_id`)\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\"" +
")\n" +
"AS\n" +
"SELECT \n" +
");";
starRocksAssert.withTable(tableSQL);
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds\n" +
"date_trunc('day', `dt`) AS ds\n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);
{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('month', `dt`) = '2023-04-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('minute', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('minute', `dt`) BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"WHERE date_trunc('month', `dt`) BETWEEN '2023-04-01' AND '2023-05-01'\n" +
"group by ds")
.match("test_partition_expr_mv1");
.nonMatch("test_partition_expr_mv1");
}

{
Expand All @@ -279,7 +344,126 @@ public void testDateTruncPartitionColumnExpr() throws Exception {
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.match("test_partition_expr_mv1");
.nonMatch("test_partition_expr_mv1");
}

starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
starRocksAssert.dropTable("test_partition_expr_tbl1");
}

@Test
public void testDateTruncPartitionColumnExpr3() throws Exception {
String tableSQL = "CREATE TABLE `test_partition_expr_tbl1` (\n" +
" `order_id` bigint(20) NOT NULL,\n" +
" `dt` datetime NOT NULL,\n" +
" `value` varchar(256) NULL \n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`order_id`, `dt`)\n" +
"PARTITION BY RANGE(`dt`)\n" +
"(\n" +
"PARTITION p2023041017 VALUES [(\"2023-04-10 17:00:00\"), (\"2023-04-10 18:00:00\")),\n" +
"PARTITION p2023041021 VALUES [(\"2023-04-10 21:00:00\"), (\"2023-04-10 22:00:00\"))\n" +
")\n" +
"DISTRIBUTED BY HASH(`order_id`)\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\"" +
");";
starRocksAssert.withTable(tableSQL);
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"date_trunc('hour', `dt`) AS ds,\n" +
"bitmap_union(to_bitmap(`order_id`)) AS `order_num` \n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) = '2023-04-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}
{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) BETWEEN '2023-04-01' AND '2023-05-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
starRocksAssert.dropTable("test_partition_expr_tbl1");
}

@Test
public void testDateTruncPartitionColumnExpr4() throws Exception {
String tableSQL = "CREATE TABLE `test_partition_expr_tbl1` (\n" +
" `order_id` bigint(20) NOT NULL,\n" +
" `dt` datetime NOT NULL,\n" +
" `value` varchar(256) NULL \n" +
") ENGINE=OLAP\n" +
"DUPLICATE KEY(`order_id`, `dt`)\n" +
"PARTITION BY date_trunc('hour', `dt`)\n" + // with date_trunc partition expression
"DISTRIBUTED BY HASH(`order_id`)\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\"" +
");";
starRocksAssert.withTable(tableSQL);
String mv = "CREATE MATERIALIZED VIEW `test_partition_expr_mv1`\n" +
"PARTITION BY ds \n" +
"DISTRIBUTED BY RANDOM \n" +
"AS SELECT \n" +
"date_trunc('hour', `dt`) AS ds,\n" +
"bitmap_union(to_bitmap(`order_id`)) AS `order_num` \n" +
"FROM `test_partition_expr_tbl1`\n" +
"group by ds;";
starRocksAssert.withMaterializedView(mv);

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) = '2023-04-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE date_trunc('day', `dt`) BETWEEN '2023-04-01' AND '2023-05-01'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

{
sql("SELECT \n" +
"count(DISTINCT `order_id`) AS `order_num`, \n" +
"date_trunc('day', `dt`) AS ds \n" +
"FROM `test_partition_expr_tbl1`\n" +
"WHERE `dt` BETWEEN '2023-04-11' AND '2023-04-12'\n" +
"group by ds")
.nonMatch("test_partition_expr_mv1");
}

starRocksAssert.dropMaterializedView("test_partition_expr_mv1");
Expand Down
8 changes: 4 additions & 4 deletions ...ocks/sql/optimizer/rule/transformation/materialization/MvRewritePartialPartitionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -817,7 +817,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr1() throws Exception {

{
String query = "select date_trunc('minute', `k1`) AS ds, sum(v1) " +
" FROM base_tbl1 where `k1` = '2020-02-11' group by ds";
" FROM base_tbl1 where date_trunc('minute', `k1`) = '2020-02-11' group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "ds = '2020-02-11 00:00:00'");
}
Expand Down Expand Up @@ -888,7 +888,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr2() throws Exception {

{
String query = "select date_trunc('minute', `k1`) AS ds, sum(v1) " +
" FROM base_tbl1 where `k1` = '2020-02-11' group by ds";
" FROM base_tbl1 where date_trunc('minute', `k1`) = '2020-02-11' group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "ds = '2020-02-11 00:00:00'");
}
Expand Down Expand Up @@ -923,7 +923,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr2() throws Exception {
" WHERE date_trunc('minute', `k1`) <= '2020-03-01 00:00:00' " +
" group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "UNION");
PlanTestBase.assertNotContains(plan, "test_mv1", "UNION");
}

{
Expand All @@ -933,7 +933,7 @@ public void testPartialPartitionRewriteWithDateTruncExpr2() throws Exception {
" date_trunc('minute', `k1`) <= '2020-03-01 00:00:00' " +
" group by ds";
String plan = getFragmentPlan(query);
PlanTestBase.assertContains(plan, "test_mv1", "UNION");
PlanTestBase.assertNotContains(plan, "test_mv1", "UNION");
}

dropMv("test", "test_mv1");
Expand Down

0 comments on commit b455786

Please sign in to comment.