From c021d6b38159debc72f271f2b6e6626955758c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E7=95=85=2826526535=29?= Date: Mon, 26 Dec 2022 14:51:04 +0800 Subject: [PATCH 1/3] Doris support deducing partitions https://github.com/bytedance/bitsail/issues/149 --- .../bitsail/common/util/DateUtil.java | 27 ++++++++--- .../bitsail/common/util/DateUtilTest.java | 26 +++++++++++ .../doris/option/DorisWriterOptions.java | 6 +-- .../partition/DorisPartitionTemplate.java | 25 +++++++++++ .../connector/doris/sink/DorisSink.java | 45 ++++++++++++++----- 5 files changed, 110 insertions(+), 19 deletions(-) create mode 100644 bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java index abbddb2e0..5fe7bc61f 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java @@ -26,11 +26,7 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.TimeZone; +import java.util.*; public class DateUtil { public static final int LENGTH_SECOND = 10; @@ -249,4 +245,25 @@ public static long convertStringToSeconds(Column column) { return stringToDate(column.asString(), null, null).toInstant().getEpochSecond(); } } + + public static List getDatesBetweenTwoDate(Date beginDate, Date endDate) { + List dataRange = new ArrayList<>(); + dataRange.add(beginDate); + Calendar cal = Calendar.getInstance(); + cal.setTime(beginDate); + while (endDate.after(cal.getTime())) { + cal.add(Calendar.DAY_OF_MONTH, 1); + dataRange.add(cal.getTime()); + } + return dataRange; + } + + public static Date getNDaysAfterDate(Date date, int n){ + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.DAY_OF_MONTH, n); + return cal.getTime(); + } + + } diff --git a/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java b/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java index 844837bac..5fb3ae70e 100644 --- a/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java +++ b/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java @@ -21,6 +21,10 @@ import org.junit.Assert; import org.junit.Test; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + public class DateUtilTest { @Test public void testConvertStringToSeconds() { @@ -29,4 +33,26 @@ public void testConvertStringToSeconds() { Assert.assertEquals(1575734400L, DateUtil.convertStringToSeconds(new StringColumn("2020-08-01"))); Assert.assertEquals(1596214923L, DateUtil.convertStringToSeconds(new StringColumn("2020-08-01 01:02:03"))); } + + @Test + public void testGetDatesBetweenTwoDate(){ + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + try { + Date dStart = sdf.parse("2022-12-26"); + Date dEnd = sdf.parse("2022-12-26"); + Assert.assertEquals(1, DateUtil.getDatesBetweenTwoDate(dStart, dEnd).size()); + } catch (ParseException e) { + e.printStackTrace(); + } + } + @Test + public void testGetNDaysAfterDate(){ + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + try { + Date date = sdf.parse("2022-12-31"); + Assert.assertEquals("2023-01-01", sdf.format(DateUtil.getNDaysAfterDate(date, 1))); + } catch (ParseException e) { + e.printStackTrace(); + } + } } diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java index 23e0bac96..019dd57c9 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java @@ -21,8 +21,8 @@ import com.bytedance.bitsail.common.option.WriterOptions; import com.alibaba.fastjson.TypeReference; +import com.bytedance.bitsail.connector.doris.partition.DorisPartitionTemplate; -import java.util.List; import java.util.Map; import static com.bytedance.bitsail.common.option.ConfigOptions.key; @@ -64,9 +64,9 @@ public interface DorisWriterOptions extends WriterOptions.BaseWriterOptions { key(WRITER_PREFIX + "table_has_partition") .defaultValue(true); - ConfigOption>> PARTITIONS = + ConfigOption PARTITIONS = key(WRITER_PREFIX + "partitions") - .onlyReference(new TypeReference>>() {}); + .onlyReference(new TypeReference() {}); ConfigOption SINK_FLUSH_INTERVAL_MS = key(WRITER_PREFIX + "sink_flush_interval_ms") diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java new file mode 100644 index 000000000..4a8c47711 --- /dev/null +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java @@ -0,0 +1,25 @@ +package com.bytedance.bitsail.connector.doris.partition; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class DorisPartitionTemplate { + + @JsonProperty(value = "prefix", required = false, defaultValue = "p") + private String prefix; + + @JsonProperty(value = "start_range", required = true) + private String startRange; + + @JsonProperty(value = "end_range", required = true) + private String endRange; + + @JsonProperty(value = "pattern", required = false, defaultValue = "yyyy-MM-dd") + private String pattern; +} diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java index 0b65c1db1..57553f7eb 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java @@ -21,11 +21,13 @@ import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; import com.bytedance.bitsail.base.serializer.BinarySerializer; import com.bytedance.bitsail.base.serializer.SimpleBinarySerializer; +import com.bytedance.bitsail.common.BitSailException; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.exception.CommonErrorCode; import com.bytedance.bitsail.common.model.ColumnInfo; import com.bytedance.bitsail.common.type.TypeInfoConverter; import com.bytedance.bitsail.common.type.filemapping.FileMappingTypeInfoConverter; +import com.bytedance.bitsail.common.util.DateUtil; import com.bytedance.bitsail.connector.doris.DorisConnectionHolder; import com.bytedance.bitsail.connector.doris.committer.DorisCommittable; import com.bytedance.bitsail.connector.doris.committer.DorisCommittableSerializer; @@ -36,6 +38,7 @@ import com.bytedance.bitsail.connector.doris.option.DorisWriterOptions; import com.bytedance.bitsail.connector.doris.partition.DorisPartition; import com.bytedance.bitsail.connector.doris.partition.DorisPartitionManager; +import com.bytedance.bitsail.connector.doris.partition.DorisPartitionTemplate; import com.bytedance.bitsail.connector.doris.sink.ddl.DorisSchemaManagerGenerator; import com.alibaba.fastjson.JSON; @@ -44,11 +47,9 @@ import java.io.IOException; import java.sql.SQLException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; import java.util.stream.Collectors; public class DorisSink implements Sink { @@ -137,16 +138,38 @@ private void initDorisOptions(BitSailConfiguration writerConfiguration) { // Need partition info in batch replace modes. if (isHasPartition && this.writeMode.equals(DorisExecutionOptions.WRITE_MODE.BATCH_REPLACE)) { //BATCH and REPLACE mode need the partition infos - List> partitionList = writerConfiguration.getNecessaryOption(DorisWriterOptions.PARTITIONS, CommonErrorCode.CONFIG_ERROR); - builder.partitions( - partitionList.stream() - .map(partition -> JSON.parseObject(JSON.toJSONString(partition), DorisPartition.class)) - .collect(Collectors.toList()) - ); + DorisPartitionTemplate dorisPartitionTemplate = writerConfiguration.getNecessaryOption(DorisWriterOptions.PARTITIONS, CommonErrorCode.CONFIG_ERROR); + List dorisPartitions = parseTemplateToDorisPartitions(dorisPartitionTemplate); + builder.partitions(dorisPartitions); } dorisOptions = builder.build(); } + private List parseTemplateToDorisPartitions(DorisPartitionTemplate dorisPartitionTemplate){ + String pattern = dorisPartitionTemplate.getPattern(); + String prefix = dorisPartitionTemplate.getPrefix(); + String start = dorisPartitionTemplate.getStartRange(); + String end = dorisPartitionTemplate.getEndRange(); + + SimpleDateFormat sdf = new SimpleDateFormat(pattern); + try { + Date dStart = sdf.parse(start); + Date dEnd = sdf.parse(end); + List dorisPartitions = new ArrayList<>(); + List listDate = DateUtil.getDatesBetweenTwoDate(dStart, dEnd); + listDate.forEach(date->{ + DorisPartition dorisPartition = new DorisPartition(); + dorisPartition.setName(prefix + sdf.format(date)); + dorisPartition.setStartRange(Collections.singletonList(sdf.format(date))); + dorisPartition.setEndRange(Collections.singletonList(sdf.format(DateUtil.getNDaysAfterDate(date, 1)))); + dorisPartitions.add(dorisPartition); + }); + return dorisPartitions; + } catch (ParseException e) { + throw BitSailException.asBitSailException(CommonErrorCode.CONFIG_ERROR, "Can't parse configuration info: " + dorisPartitionTemplate, e); + } + } + private void initDorisExecutionOptions(BitSailConfiguration writerConfiguration) { LOG.info("Start to init DorisExecutionOptions!"); final DorisExecutionOptions.DorisExecutionOptionsBuilder builder = DorisExecutionOptions.builder(); From 37a4631c4e88dca6fb07d6aa1801cd09252e0e57 Mon Sep 17 00:00:00 2001 From: beyond-up <3451663959@qq.com> Date: Mon, 26 Dec 2022 16:41:09 +0800 Subject: [PATCH 2/3] modified after checkstyle --- .../bitsail/common/util/DateUtil.java | 11 +++++--- .../bitsail/common/util/DateUtilTest.java | 26 ++++--------------- .../doris/option/DorisWriterOptions.java | 2 +- .../partition/DorisPartitionTemplate.java | 18 ++++++------- .../connector/doris/sink/DorisSink.java | 20 +++++++++----- 5 files changed, 36 insertions(+), 41 deletions(-) diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java index 5fe7bc61f..e45803526 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/util/DateUtil.java @@ -26,7 +26,13 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; public class DateUtil { public static final int LENGTH_SECOND = 10; @@ -258,12 +264,11 @@ public static List getDatesBetweenTwoDate(Date beginDate, Date endDate) { return dataRange; } - public static Date getNDaysAfterDate(Date date, int n){ + public static Date getNDaysAfterDate(Date date, int n) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.add(Calendar.DAY_OF_MONTH, n); return cal.getTime(); } - } diff --git a/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java b/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java index 5fb3ae70e..efa4f4f47 100644 --- a/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java +++ b/bitsail-common/src/test/java/com/bytedance/bitsail/common/util/DateUtilTest.java @@ -21,8 +21,6 @@ import org.junit.Assert; import org.junit.Test; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Date; public class DateUtilTest { @@ -35,24 +33,10 @@ public void testConvertStringToSeconds() { } @Test - public void testGetDatesBetweenTwoDate(){ - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - try { - Date dStart = sdf.parse("2022-12-26"); - Date dEnd = sdf.parse("2022-12-26"); - Assert.assertEquals(1, DateUtil.getDatesBetweenTwoDate(dStart, dEnd).size()); - } catch (ParseException e) { - e.printStackTrace(); - } - } - @Test - public void testGetNDaysAfterDate(){ - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - try { - Date date = sdf.parse("2022-12-31"); - Assert.assertEquals("2023-01-01", sdf.format(DateUtil.getNDaysAfterDate(date, 1))); - } catch (ParseException e) { - e.printStackTrace(); - } + public void testGetDatesBetweenTwoDate() { + Date start = new Date(); + Date end = new Date(); + Assert.assertEquals(1, DateUtil.getDatesBetweenTwoDate(start, end).size()); } + } diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java index 019dd57c9..ced88a29c 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/option/DorisWriterOptions.java @@ -19,9 +19,9 @@ import com.bytedance.bitsail.common.annotation.Essential; import com.bytedance.bitsail.common.option.ConfigOption; import com.bytedance.bitsail.common.option.WriterOptions; +import com.bytedance.bitsail.connector.doris.partition.DorisPartitionTemplate; import com.alibaba.fastjson.TypeReference; -import com.bytedance.bitsail.connector.doris.partition.DorisPartitionTemplate; import java.util.Map; diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java index 4a8c47711..14e6d7f87 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java @@ -5,21 +5,21 @@ import lombok.Data; import lombok.NoArgsConstructor; - @AllArgsConstructor @NoArgsConstructor @Data public class DorisPartitionTemplate { - @JsonProperty(value = "prefix", required = false, defaultValue = "p") - private String prefix; + @JsonProperty(value = "prefix", required = false, defaultValue = "p") + private String prefix; + + @JsonProperty(value = "start_range", required = true) + private String startRange; - @JsonProperty(value = "start_range", required = true) - private String startRange; + @JsonProperty(value = "end_range", required = true) + private String endRange; - @JsonProperty(value = "end_range", required = true) - private String endRange; + @JsonProperty(value = "pattern", required = false, defaultValue = "yyyy-MM-dd") + private String pattern; - @JsonProperty(value = "pattern", required = false, defaultValue = "yyyy-MM-dd") - private String pattern; } diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java index 57553f7eb..85b0f3750 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java @@ -49,8 +49,14 @@ import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; public class DorisSink implements Sink { private static final Logger LOG = LoggerFactory.getLogger(DorisSink.class); @@ -145,7 +151,7 @@ private void initDorisOptions(BitSailConfiguration writerConfiguration) { dorisOptions = builder.build(); } - private List parseTemplateToDorisPartitions(DorisPartitionTemplate dorisPartitionTemplate){ + private List parseTemplateToDorisPartitions(DorisPartitionTemplate dorisPartitionTemplate) { String pattern = dorisPartitionTemplate.getPattern(); String prefix = dorisPartitionTemplate.getPrefix(); String start = dorisPartitionTemplate.getStartRange(); @@ -153,11 +159,11 @@ private List parseTemplateToDorisPartitions(DorisPartitionTempla SimpleDateFormat sdf = new SimpleDateFormat(pattern); try { - Date dStart = sdf.parse(start); - Date dEnd = sdf.parse(end); + Date dateStart = sdf.parse(start); + Date dateEnd = sdf.parse(end); List dorisPartitions = new ArrayList<>(); - List listDate = DateUtil.getDatesBetweenTwoDate(dStart, dEnd); - listDate.forEach(date->{ + List listDate = DateUtil.getDatesBetweenTwoDate(dateStart, dateEnd); + listDate.forEach(date -> { DorisPartition dorisPartition = new DorisPartition(); dorisPartition.setName(prefix + sdf.format(date)); dorisPartition.setStartRange(Collections.singletonList(sdf.format(date))); From 9be0a282491eca6cc1a5de31899094a69a70d8af Mon Sep 17 00:00:00 2001 From: beyond-up <3451663959@qq.com> Date: Fri, 24 Feb 2023 17:03:00 +0800 Subject: [PATCH 3/3] merge master --- .../doris/partition/DorisPartitionLevel.java | 31 +++++++++++++++++++ .../partition/DorisPartitionTemplate.java | 26 ++++++++++++++++ .../connector/doris/sink/DorisSink.java | 23 ++++++++++++-- 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionLevel.java diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionLevel.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionLevel.java new file mode 100644 index 000000000..9f9aa8a12 --- /dev/null +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionLevel.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.bytedance.bitsail.connector.doris.partition; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum DorisPartitionLevel { + DAY("day"), + WEEK("week"), + MONTH("month"), + QUARTER("quarter"); + + private final String type; +} diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java index 14e6d7f87..dd5715c62 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/partition/DorisPartitionTemplate.java @@ -1,3 +1,19 @@ +/* + * Copyright 2022 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.bytedance.bitsail.connector.doris.partition; import com.fasterxml.jackson.annotation.JsonProperty; @@ -22,4 +38,14 @@ public class DorisPartitionTemplate { @JsonProperty(value = "pattern", required = false, defaultValue = "yyyy-MM-dd") private String pattern; + @JsonProperty(value = "partition_level", required = false, defaultValue = "day") + private String partitionLevel; + + public enum DorisPartitionLevel { + DAY, + WEEK, + MONTH, + QUARTER; + } + } diff --git a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java index c6b34fa2f..4710aad46 100644 --- a/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java +++ b/bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/sink/DorisSink.java @@ -21,7 +21,6 @@ import com.bytedance.bitsail.base.connector.writer.v1.WriterCommitter; import com.bytedance.bitsail.base.serializer.BinarySerializer; import com.bytedance.bitsail.base.serializer.SimpleVersionedBinarySerializer; -import com.bytedance.bitsail.base.serializer.SimpleBinarySerializer; import com.bytedance.bitsail.common.BitSailException; import com.bytedance.bitsail.common.configuration.BitSailConfiguration; import com.bytedance.bitsail.common.exception.CommonErrorCode; @@ -156,13 +155,33 @@ private List parseTemplateToDorisPartitions(DorisPartitionTempla String prefix = dorisPartitionTemplate.getPrefix(); String start = dorisPartitionTemplate.getStartRange(); String end = dorisPartitionTemplate.getEndRange(); + DorisPartitionTemplate.DorisPartitionLevel partitionLevel = DorisPartitionTemplate.DorisPartitionLevel.valueOf( + dorisPartitionTemplate.getPartitionLevel().toUpperCase()); SimpleDateFormat sdf = new SimpleDateFormat(pattern); try { Date dateStart = sdf.parse(start); Date dateEnd = sdf.parse(end); List dorisPartitions = new ArrayList<>(); - List listDate = DateUtil.getDatesBetweenTwoDate(dateStart, dateEnd); + List listDate; + switch (partitionLevel) { + case DAY: + listDate = DateUtil.getDatesBetweenTwoDate(dateStart, dateEnd); + break; + case WEEK: + listDate = DateUtil.getDatesBetweenTwoDate(dateStart, dateEnd); + break; + case MONTH: + listDate = DateUtil.getDatesBetweenTwoDate(dateStart, dateEnd); + break; + case QUARTER: + listDate = DateUtil.getDatesBetweenTwoDate(dateStart, dateEnd); + break; + default: + throw BitSailException.asBitSailException(CommonErrorCode.CONFIG_ERROR, "The configure partition level " + partitionLevel + + " is not supported, only support day, week, month and quarter"); + } + listDate.forEach(date -> { DorisPartition dorisPartition = new DorisPartition(); dorisPartition.setName(prefix + sdf.format(date));