From 368ae08ddb3d7669ddd099e3c2a8880682a76e79 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 9 Oct 2023 11:14:47 +0530 Subject: [PATCH] test cases --- .../msq/test/CalciteSelectQueryMSQTest.java | 1 - .../msq/test/CalciteUnionQueryMSQTest.java | 12 +- .../apache/druid/query/UnionDataSource.java | 22 +-- .../druid/sql/calcite/CalciteQueryTest.java | 101 +++++++++++++ .../sql/calcite/CalciteUnionQueryTest.java | 141 ------------------ 5 files changed, 122 insertions(+), 155 deletions(-) diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java index f9f9986a0317..5ee3ba875388 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteSelectQueryMSQTest.java @@ -174,5 +174,4 @@ public void testArrayAggQueryOnComplexDatatypes() ); } } - } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteUnionQueryMSQTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteUnionQueryMSQTest.java index 90878d8e3f1c..8ee9e78c8388 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteUnionQueryMSQTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteUnionQueryMSQTest.java @@ -111,15 +111,19 @@ protected QueryTestBuilder testBuilder() } /** - * Doesn't pass through Druid however the planning error is different as it rewrites to a union datasource. - * This test is disabled because MSQ wants to support union datasources, and it makes little sense to add highly - * conditional planning error for the same. Planning errors are merely hints, and this is one of those times - * when the hint is incorrect till MSQ starts supporting the union datasource. + * Generates a different error hint than what is required by the native engine, since planner does try to plan "UNION" + * using group by, however fails due to the column name mismatch. + * MSQ does wnat to support any type of data source, with least restrictive column names and types, therefore it + * should eventually work. */ @Test @Override public void testUnionIsUnplannable() { + assertQueryIsUnplannable( + "SELECT dim2, dim1, m1 FROM foo2 UNION SELECT dim1, dim2, m1 FROM foo", + "SQL requires union between two tables and column names queried for each table are different Left: [dim2, dim1, m1], Right: [dim1, dim2, m1]." + ); } diff --git a/processing/src/main/java/org/apache/druid/query/UnionDataSource.java b/processing/src/main/java/org/apache/druid/query/UnionDataSource.java index f28f594de28c..27a0113d76f1 100644 --- a/processing/src/main/java/org/apache/druid/query/UnionDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/UnionDataSource.java @@ -23,12 +23,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.segment.SegmentReference; +import org.apache.druid.utils.CollectionUtils; import java.util.Collections; import java.util.List; @@ -75,14 +75,18 @@ public List getDataSources() @Override public Set getTableNames() { - return dataSources.stream() - .map(input -> { - if (!(input instanceof TableDataSource)) { - throw DruidException.defensive("should be table"); - } - return Iterables.getOnlyElement(input.getTableNames()); - }) - .collect(Collectors.toSet()); + return dataSources + .stream() + .map(input -> { + if (!(input instanceof TableDataSource)) { + throw DruidException.defensive("should be table"); + } + return CollectionUtils.getOnlyElement( + input.getTableNames(), + xs -> DruidException.defensive("Expected only single table name in TableDataSource") + ); + }) + .collect(Collectors.toSet()); } /** diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 623f7f0b7ff3..735b1f0f49b7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -42,6 +42,7 @@ import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryDataSource; import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.UnionDataSource; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory; @@ -3197,6 +3198,106 @@ public void testNullFloatFilter() ); } + /** + * This test case should be in {@link CalciteUnionQueryTest}. However, there's a bug in the test framework that + * doesn't reset framework once the merge buffers + */ + @DecoupledIgnore + @Test + public void testUnionAllSameTableThreeTimes() + { + testQuery( + "SELECT\n" + + "dim1, dim2, SUM(m1), COUNT(*)\n" + + "FROM (SELECT * FROM foo UNION ALL SELECT * FROM foo UNION ALL SELECT * FROM foo)\n" + + "WHERE dim2 = 'a' OR dim2 = 'def'\n" + + "GROUP BY 1, 2", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource( + new UnionDataSource( + ImmutableList.of( + new TableDataSource(CalciteTests.DATASOURCE1), + new TableDataSource(CalciteTests.DATASOURCE1), + new TableDataSource(CalciteTests.DATASOURCE1) + ) + ) + ) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setDimFilter(in("dim2", ImmutableList.of("def", "a"), null)) + .setDimensions( + new DefaultDimensionSpec("dim1", "d0"), + new DefaultDimensionSpec("dim2", "d1") + ) + .setAggregatorSpecs( + aggregators( + new DoubleSumAggregatorFactory("a0", "m1"), + new CountAggregatorFactory("a1") + ) + ) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"", "a", 3.0, 3L}, + new Object[]{"1", "a", 12.0, 3L} + ) + ); + } + + @Test + public void testExactCountDistinctUsingSubqueryOnUnionAllTables() + { + testQuery( + "SELECT\n" + + " SUM(cnt),\n" + + " COUNT(*)\n" + + "FROM (\n" + + " SELECT dim2, SUM(cnt) AS cnt\n" + + " FROM (SELECT * FROM druid.foo UNION ALL SELECT * FROM druid.foo)\n" + + " GROUP BY dim2\n" + + ")", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource( + new QueryDataSource( + GroupByQuery.builder() + .setDataSource( + new UnionDataSource( + ImmutableList.of( + new TableDataSource(CalciteTests.DATASOURCE1), + new TableDataSource(CalciteTests.DATASOURCE1) + ) + ) + ) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ) + ) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setAggregatorSpecs(aggregators( + new LongSumAggregatorFactory("_a0", "a0"), + new CountAggregatorFactory("_a1") + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + NullHandling.replaceWithDefault() ? + ImmutableList.of( + new Object[]{12L, 3L} + ) : + ImmutableList.of( + new Object[]{12L, 4L} + ) + ); + } + @Test public void testNullDoubleTopN() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteUnionQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteUnionQueryTest.java index 1e1ec3d6986c..773e1776857d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteUnionQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteUnionQueryTest.java @@ -20,15 +20,12 @@ package org.apache.druid.sql.calcite; import com.google.common.collect.ImmutableList; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.QueryDataSource; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.UnionDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.sql.calcite.filtration.Filtration; @@ -345,49 +342,6 @@ public void testUnionAllSameTableTwiceWithDifferentMapping() ); } - @Test - public void testUnionAllSameTableThreeTimes() - { - testQuery( - "SELECT\n" - + "dim1, dim2, SUM(m1), COUNT(*)\n" - + "FROM (SELECT * FROM foo UNION ALL SELECT * FROM foo UNION ALL SELECT * FROM foo)\n" - + "WHERE dim2 = 'a' OR dim2 = 'def'\n" - + "GROUP BY 1, 2", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource( - new UnionDataSource( - ImmutableList.of( - new TableDataSource(CalciteTests.DATASOURCE1), - new TableDataSource(CalciteTests.DATASOURCE1), - new TableDataSource(CalciteTests.DATASOURCE1) - ) - ) - ) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setDimFilter(in("dim2", ImmutableList.of("def", "a"), null)) - .setDimensions( - new DefaultDimensionSpec("dim1", "d0"), - new DefaultDimensionSpec("dim2", "d1") - ) - .setAggregatorSpecs( - aggregators( - new DoubleSumAggregatorFactory("a0", "m1"), - new CountAggregatorFactory("a1") - ) - ) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"", "a", 3.0, 3L}, - new Object[]{"1", "a", 12.0, 3L} - ) - ); - } - @Test public void testUnionAllThreeTablesColumnCountMismatch1() { @@ -448,99 +402,4 @@ public void testUnionAllThreeTablesColumnCountMismatch3() } } - @Test - public void testUnionAllSameTableThreeTimesWithSameMapping() - { - testQuery( - "SELECT\n" - + "dim1, dim2, SUM(m1), COUNT(*)\n" - + "FROM (SELECT dim1, dim2, m1 FROM foo UNION ALL SELECT dim1, dim2, m1 FROM foo UNION ALL SELECT dim1, dim2, m1 FROM foo)\n" - + "WHERE dim2 = 'a' OR dim2 = 'def'\n" - + "GROUP BY 1, 2", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource( - new UnionDataSource( - ImmutableList.of( - new TableDataSource(CalciteTests.DATASOURCE1), - new TableDataSource(CalciteTests.DATASOURCE1), - new TableDataSource(CalciteTests.DATASOURCE1) - ) - ) - ) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setDimFilter(in("dim2", ImmutableList.of("def", "a"), null)) - .setDimensions( - new DefaultDimensionSpec("dim1", "d0"), - new DefaultDimensionSpec("dim2", "d1") - ) - .setAggregatorSpecs( - aggregators( - new DoubleSumAggregatorFactory("a0", "m1"), - new CountAggregatorFactory("a1") - ) - ) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{"", "a", 3.0, 3L}, - new Object[]{"1", "a", 12.0, 3L} - ) - ); - } - - @Test - public void testExactCountDistinctUsingSubqueryOnUnionAllTables() - { - testQuery( - "SELECT\n" - + " SUM(cnt),\n" - + " COUNT(*)\n" - + "FROM (\n" - + " SELECT dim2, SUM(cnt) AS cnt\n" - + " FROM (SELECT * FROM druid.foo UNION ALL SELECT * FROM druid.foo)\n" - + " GROUP BY dim2\n" - + ")", - ImmutableList.of( - GroupByQuery.builder() - .setDataSource( - new QueryDataSource( - GroupByQuery.builder() - .setDataSource( - new UnionDataSource( - ImmutableList.of( - new TableDataSource(CalciteTests.DATASOURCE1), - new TableDataSource(CalciteTests.DATASOURCE1) - ) - ) - ) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))) - .setAggregatorSpecs(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ) - ) - .setInterval(querySegmentSpec(Filtration.eternity())) - .setGranularity(Granularities.ALL) - .setAggregatorSpecs(aggregators( - new LongSumAggregatorFactory("_a0", "a0"), - new CountAggregatorFactory("_a1") - )) - .setContext(QUERY_CONTEXT_DEFAULT) - .build() - ), - NullHandling.replaceWithDefault() ? - ImmutableList.of( - new Object[]{12L, 3L} - ) : - ImmutableList.of( - new Object[]{12L, 4L} - ) - ); - } - }