5.4.19-20240428_17142882 release

README.md

@@ -116,7 +116,7 @@ You are welcome to make contributions to PolarDB-X. We appreciate all the contri
 ## Community
 You can join these groups and chats to discuss and ask PolarDB-X related questions:
  - DingTalk Group: [32432897](https://h5.dingtalk.com/circle/healthCheckin.html?dtaction=os&corpId=dingc5456617ca6ab502e1cc01e222598659&1b3d4=1ec1b&cbdbhh=qwertyuiop#/) 
- ![DingTalk Group](docs/images/dingtalk_group.png)
+ ![DingTalk Group](docs/images/dingtalk_group.jpg)
  - WeChat Group: 阿里云 PolarDB-X 开源交流群 (Contact group manager to get into wechat group. Managers' ID: oldbread3, hustfxj, agapple0002) 
  ![WeChat Manager 1](docs/images/wechat_manager_a.jpg) ![WeChat Manager 2](docs/images/wechat_manager_b.jpg) ![WeChat Manager 3](docs/images/wechat_manager_c.jpg)
 

codestyle/format_lab.sh

@@ -2,7 +2,7 @@
 
 baseDir=`pwd`/
 commit=`git merge-base origin/polardbx_opensource HEAD`
-files=`git diff --name-only ${commit} | grep '.java' | grep -v 'polardbx-calcite' | grep -v 'polardbx-rpc/src/main/java/com/mysql/cj'| grep -v 'com/alibaba/polardbx/rpc/cdc'| xargs -I {} echo ${baseDir}{}`
+files=`git diff --name-only ${commit} | grep '.java' | grep -v 'polardbx-calcite' | grep -v 'polardbx-orc' | grep -v 'polardbx-rpc/src/main/java/com/mysql/cj'| grep -v 'com/alibaba/polardbx/rpc/cdc'| xargs -I {} echo ${baseDir}{}`
 count=0
 batchFile=''
 for file in $files; do

docs/en/quickstart-development.md

@@ -80,12 +80,13 @@ make install
 
 ```shell
 # install JDK 1.8 and Maven 3
-# enter the polardbx-sql directory
-cd polardbx-sql/
 
 # make sure polardbx-rpc (polardbx-glue) initialized
 git submodule update --init
 
+# enter the polardbx-sql directory 
+cd polardbx-sql/
+
 # compile&install
 mvn install -D maven.test.skip=true -D env=release 
 

docs/zh_CN/quickstart-development.md

@@ -126,7 +126,7 @@ mkdir -p /u01/my3306/{data,log,run,tmp,mysql}
 
 - metadb user：以下采用`my_polarx`
 - metadb database：创建metadb库，以下采用 `polardbx_meta_db_polardbx`
-- 密码加密key（dnPasswordKey)：以下采用 `asdf1234ghjk5678` (如果要修改加密key, 新key的长度要求为16位)
+- 密码加密key（dnPasswordKey)：以下采用 `asdf1234ghjk5678`
 - PolarDB-X默认用户名：默认为 `polarx_root`
 - PolarDB-X默认用户密码：默认为 `123456`，可通过 `-S` 参数修改
 
@@ -199,7 +199,7 @@ mysql -h127.1 -P8527 -upolardbx_root
 - metadb database：和启动PolarDB-X时设置的值保持一致，以下采用 `polardbx_meta_db_polardbx`
 - metadb password：和启动PolarDB-X时设置的值保持一致，需使用密文，以下采用`HMqvkvXZtT7XedA6t2IWY8+D7fJWIJir/mIY1Nf1b58=`
 - metadb port：和启动MySQL时设置的值保持一致，以下采用 `4886`
-- 密码加密key（dnPasswordKey)：和启动PolarDB-X时设置的值保持一致，以下采用 `asdf1234ghjk5678` (如果要修改加密key, 新key的长度要求为16位)
+- 密码加密key（dnPasswordKey)：和启动PolarDB-X时设置的值保持一致，以下采用 `asdf1234ghjk5678`
 - PolarDB-X用户名：和启动PolarDB-X时设置的值保持一致，以下采用默认值 `polardbx_root`
 - PolarDB-X用户密码：和启动PolarDB-X时设置的值保持一致，需使用密文，以下采用默认值`H1AzXc2NmCs61dNjH5nMvA==`
 - PolarDB-X端口：和启动PolarDB-X时设置的值保持一致，以下采用默认值 `8527`

polardbx-calcite/pom.xml

@@ -20,13 +20,13 @@ limitations under the License.
  <parent>
  <artifactId>polardbx</artifactId>
  <groupId>com.alibaba.polardbx</groupId>
- <version>5.4.18-SNAPSHOT</version>
+ <version>${revision}</version>
  </parent>
  <artifactId>polardbx-calcite</artifactId>
- <packaging>jar</packaging>
- <version>5.4.18-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <version>${calcite.version}</version>
+ <name>${project.artifactId} module for polardbx ${project.version}</name>
  <description>Core Calcite APIs and engine.</description>
- <name>${project.artifactId} module for polardbx ${project.version}</name>
  <properties>
  <top.dir>${project.basedir}/..</top.dir>
  <build.timestamp>${maven.build.timestamp}</build.timestamp>
@@ -57,10 +57,12 @@ limitations under the License.
  <version>${commons-dbcp.version}</version>
  </dependency>
  <dependency>
-<!-- <groupId>org.apache.commons</groupId>-->
-<!-- <artifactId>commons-lang3</artifactId>-->
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
  </dependency>
  <dependency>
  <groupId>com.fasterxml.jackson.core</groupId>

polardbx-calcite/src/main/java/org/apache/calcite/jdbc/JavaTypeFactoryImpl.java

@@ -75,7 +75,7 @@ public JavaTypeFactoryImpl(RelDataTypeSystem typeSystem) {
 
  public RelDataType createStructType(Class type) {
  final List<RelDataTypeField> list = new ArrayList<>();
- for (Field field : type.getFields()) {
+ for (Field field : type.getDeclaredFields()) {
  if (!Modifier.isStatic(field.getModifiers())) {
  // FIXME: watch out for recursion
  final Type fieldType = fieldType(field);

polardbx-calcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java

@@ -19,6 +19,7 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.CorrelationId;
 import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
+import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider;
 import org.apache.calcite.rel.metadata.MetadataFactory;
 import org.apache.calcite.rel.metadata.MetadataFactoryImpl;
 import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -150,19 +151,26 @@ public MetadataFactory getMetadataFactory() {
  * method, then use {@link RelOptRuleCall#getMetadataQuery()} instead. */
  public RelMetadataQuery getMetadataQuery() {
  if (mq == null) {
- mq = RelMetadataQuery.instance();
+ mq = buildMetaQuery();
  }
 
  RelMetadataQuery local = mq;
  if (local == null) {
  // maybe some concurrent thread call invalidateMetadataQuery
  // cache RelMetadataQuery on the stack to in case of returning null
- local = RelMetadataQuery.instance();
+ local = buildMetaQuery();
  mq = local;
  }
  return local;
  }
 
+ private RelMetadataQuery buildMetaQuery() {
+ RelMetadataProvider provider = getMetadataProvider();
+ // provider in cluster cannot be a JaninoRelMetadataProvider,
+ // for it doesn't support RelMetadataProvider.apply method
+ return RelMetadataQuery.instance(JaninoRelMetadataProvider.of(provider));
+ }
+
  /**
  * Should be called whenever the current {@link RelMetadataQuery} becomes
  * invalid. Typically invoked from {@link RelOptRuleCall#transformTo}.

polardbx-calcite/src/main/java/org/apache/calcite/plan/RelOptUtil.java

@@ -235,7 +235,7 @@ public static Set<RelOptTable> findTables(RelNode rel) {
  */
  public static List<RelOptTable> findAllTables(RelNode rel) {
  final Multimap<Class<? extends RelNode>, RelNode> nodes =
-  RelMetadataQuery.instance().getNodeTypes(rel);
+ rel.getCluster().getMetadataQuery().getNodeTypes(rel);
  final List<RelOptTable> usedTables = new ArrayList<>();
  for (Entry<Class<? extends RelNode>, Collection<RelNode>> e : nodes.asMap().entrySet()) {
  if (TableScan.class.isAssignableFrom(e.getKey())) {
@@ -259,7 +259,7 @@ public static Set<RelNode> findCorrelates(RelNode rel) {
  */
  public static List<RelNode> findAllCorrelate(RelNode rel) {
  final Multimap<Class<? extends RelNode>, RelNode> nodes =
- RelMetadataQuery.instance().getNodeTypes(rel);
+ rel.getCluster().getMetadataQuery().getNodeTypes(rel);
  final List<RelNode> usedTables = new ArrayList<>();
  for (Entry<Class<? extends RelNode>, Collection<RelNode>> e : nodes.asMap().entrySet()) {
  if (Correlate.class.isAssignableFrom(e.getKey())) {
@@ -1925,6 +1925,11 @@ public static boolean eq(
  return litmus.succeed();
  }
 
+ if (type1.getSqlTypeName() == SqlTypeName.DECIMAL
+ && type2.getSqlTypeName() == SqlTypeName.DECIMAL) {
+ return litmus.succeed();
+ }
+
  if (type1 != type2) {
  return litmus.fail("type mismatch:\n{}:\n{}\n{}:\n{}",
  desc1, type1.getFullTypeString(),
@@ -2877,7 +2882,7 @@ public static boolean containsLimit(RelNode ancestor) {
  try {
  new RelVisitor() {
  public void visit(RelNode node, int ordinal, RelNode parent) {
- if (node instanceof Sort) {
+ if (node instanceof Sort&&((Sort)node).withLimit()) {
  throw Util.FoundOne.NULL;
  }
  super.visit(node, ordinal, parent);

polardbx-calcite/src/main/java/org/apache/calcite/plan/RelTraitSet.java

@@ -20,6 +20,9 @@
 import org.apache.calcite.rel.RelCollationTraitDef;
 import org.apache.calcite.rel.RelDistribution;
 import org.apache.calcite.rel.RelDistributionTraitDef;
+import org.apache.calcite.rel.RelPartitionWise;
+import org.apache.calcite.rel.RelPartitionWiseTraitDef;
+import org.apache.calcite.rel.RelPartitionWises;
 import org.apache.calcite.util.mapping.Mappings;
 
 import com.google.common.collect.ImmutableList;
@@ -390,6 +393,10 @@ public RelTraitSet getDefaultSansConvention() {
  return (@Nullable T) getTrait(RelCollationTraitDef.INSTANCE);
  }
 
+ public <T extends RelPartitionWise> @Nullable T getPartitionWise() {
+ return (@Nullable T) getTrait(RelPartitionWiseTraitDef.INSTANCE);
+ }
+
  /**
  * Returns the size of the RelTraitSet.
  *

polardbx-calcite/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java

@@ -173,6 +173,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
 
  boolean enableDerive = true;
 
+ boolean enableColumnar = false;
+
  /**
  * Extra roots for explorations.
  */
@@ -401,7 +403,7 @@ protected void registerMaterializations() {
  this.root = null;
  this.originalRoot = null;
  if (ruleCounter != null) {
- ruleCount = ruleCounter.values().stream().reduce(0L, (a,b) -> a + b).longValue();
+ ruleCount = ruleCounter.values().stream().reduce(0L, Long::sum);
  this.ruleCounter.clear();
  }
  }
@@ -1720,7 +1722,7 @@ public RelOptCost getStartUpCost(RelNode rel, RelMetadataQuery mq) {
  return costFactory.makeInfiniteCost();
  }
  // use new mq to avoid metadata cache
- mq = RelMetadataQuery.instance();
+ mq = RelMetadataQuery.instance(mq.metadataProvider);
  // for efficiency, use RelSubsetBest Cost For CumulativeCost
  mq.setUseRelSubsetBestCostForCumulativeCost(true);
  RelOptCost cost = mq.getStartUpCost(rel);
@@ -1765,4 +1767,12 @@ public boolean isEnableDerive() {
  public void setEnableDerive(boolean enableDerive) {
  this.enableDerive = enableDerive;
  }
+
+ public boolean isEnableColumnar() {
+ return enableColumnar;
+ }
+
+ public void setEnableColumnar(boolean enableColumnar) {
+ this.enableColumnar = enableColumnar;
+ }
 }

polardbx-calcite/src/main/java/org/apache/calcite/rel/RelDistribution.java

@@ -48,6 +48,10 @@ public interface RelDistribution extends RelMultipleTrait {
  */
  @Nonnull List<Integer> getKeys();
 
+ boolean isShardWise();
+
+ @Nonnull Integer getShardCnt();
+
  RelDistribution apply(Mappings.TargetMapping mapping);
 
  /** Type of distribution. */

polardbx-calcite/src/main/java/org/apache/calcite/rel/RelDistributions.java

@@ -73,6 +73,13 @@ public static RelDistribution hash(Collection<? extends Number> numbers) {
  return RelDistributionTraitDef.INSTANCE.canonize(trait);
  }
 
+ public static RelDistribution hashOss(Collection<? extends Number> numbers, int shard) {
+ ImmutableIntList list = ImmutableIntList.copyOf(numbers);
+ RelDistributionImpl trait =
+ new RelDistributionImpl(RelDistribution.Type.HASH_DISTRIBUTED, list, shard);
+ return RelDistributionTraitDef.INSTANCE.canonize(trait);
+ }
+
  /** Creates a range distribution. */
  public static RelDistribution range(Collection<? extends Number> numbers) {
  ImmutableIntList list = ImmutableIntList.copyOf(numbers);
@@ -81,8 +88,8 @@ public static RelDistribution range(Collection<? extends Number> numbers) {
  return RelDistributionTraitDef.INSTANCE.canonize(trait);
  }
 
- public static RelDistribution of(RelDistribution.Type type, ImmutableIntList keys) {
- RelDistribution distribution = new RelDistributionImpl(type, keys);
+ public static RelDistribution of(RelDistribution.Type type, ImmutableIntList keys, Integer shardCnt) {
+ RelDistribution distribution = new RelDistributionImpl(type, keys, shardCnt);
  return RelDistributionTraitDef.INSTANCE.canonize(distribution);
  }
 
@@ -104,9 +111,12 @@ public static class RelDistributionImpl implements RelDistribution {
  private final Type type;
  private final ImmutableIntList keys;
 
+ private final Integer shardCnt;
+
  public RelDistributionImpl(Type type, ImmutableIntList keys) {
  this.type = Preconditions.checkNotNull(type);
  this.keys = ImmutableIntList.copyOf(keys);
+ this.shardCnt = 0;
 // assert type != Type.HASH_DISTRIBUTED
 // || keys.size() < 2
 // || Ordering.natural().isOrdered(keys)
@@ -116,22 +126,32 @@ public RelDistributionImpl(Type type, ImmutableIntList keys) {
  || keys.isEmpty();
  }
 
+ public RelDistributionImpl(Type type, ImmutableIntList keys, Integer shardCnt) {
+ this.type = Preconditions.checkNotNull(type);
+ this.keys = ImmutableIntList.copyOf(keys);
+ this.shardCnt = shardCnt == null ? 0 : shardCnt;
+ assert type == Type.HASH_DISTRIBUTED
+ || type == Type.RANDOM_DISTRIBUTED
+ || keys.isEmpty();
+ }
+
  @Override public int hashCode() {
- return Objects.hash(type, keys);
+ return Objects.hash(type, keys, shardCnt);
  }
 
  @Override public boolean equals(Object obj) {
  return this == obj
  || obj instanceof RelDistributionImpl
  && type == ((RelDistributionImpl) obj).type
- && keys.equals(((RelDistributionImpl) obj).keys);
+ && keys.equals(((RelDistributionImpl) obj).keys)
+ && shardCnt.equals(((RelDistributionImpl) obj).shardCnt);
  }
 
  @Override public String toString() {
  if (keys.isEmpty()) {
  return type.shortName;
  } else {
- return type.shortName + keys;
+ return shardCnt > 0 ? type.shortName + keys + shardCnt : type.shortName + keys ;
  }
  }
 
@@ -143,6 +163,14 @@ public RelDistributionImpl(Type type, ImmutableIntList keys) {
  return keys;
  }
 
+ public boolean isShardWise() {
+ return shardCnt > 0;
+ }
+
+ @Nonnull public Integer getShardCnt() {
+ return shardCnt;
+ }
+
  public RelDistributionTraitDef getTraitDef() {
  return RelDistributionTraitDef.INSTANCE;
  }
@@ -158,7 +186,7 @@ public RelDistribution apply(Mappings.TargetMapping mapping) {
  }
  List<Integer> mappedKeys0 = Mappings.apply2((Mapping) mapping, keys);
  ImmutableIntList mappedKeys = normalizeKeys(mappedKeys0);
- return of(type, mappedKeys);
+ return of(type, mappedKeys, shardCnt);
  }
 
  public boolean satisfies(RelTrait trait) {
@@ -172,7 +200,7 @@ public boolean satisfies(RelTrait trait) {
  case HASH_DISTRIBUTED:
  // The "leading edge" property of Range does not apply to Hash.
  // Only Hash[x, y] satisfies Hash[x, y].
- return keys.equals(distribution.keys);
+ return keys.equals(distribution.keys) && shardCnt.equals(distribution.shardCnt);
  case RANGE_DISTRIBUTED:
  // Range[x, y] satisfies Range[x, y, z] but not Range[x]
  return Util.startsWith(distribution.keys, keys);
@@ -203,7 +231,11 @@ public void register(RelOptPlanner planner) {
  if (type == distribution.getType()
  && (type == Type.HASH_DISTRIBUTED
  || type == Type.RANGE_DISTRIBUTED)) {
- return ORDERING.compare(getKeys(), distribution.getKeys());
+ int order = ORDERING.compare(getKeys(), distribution.getKeys());
+ if (order == 0) {
+ return Integer.compare(getShardCnt(), distribution.getShardCnt());
+ }
+ return order;
  }
 
  return type.compareTo(distribution.getType());

polardbx-calcite/src/main/java/org/apache/calcite/rel/RelInput.java

@@ -104,6 +104,8 @@ public interface RelInput {
 
  RelDistribution getDistribution();
 
+ RelPartitionWise getPartitionWise();
+
  ImmutableList<ImmutableList<RexLiteral>> getTuples(String tag);
 
  ImmutableList<ImmutableList<RexNode>> getDynamicTuples(String tag);

polardbx-calcite/src/main/java/org/apache/calcite/rel/RelPartitionWise.java

@@ -0,0 +1,13 @@
+package org.apache.calcite.rel;
+
+import org.apache.calcite.plan.RelMultipleTrait;
+
+public interface RelPartitionWise extends RelMultipleTrait {
+
+ //~ Methods ----------------------------------------------------------------
+ boolean isLocalPartition();
+
+ boolean isRemotePartition();
+
+ int getCode();
+}