Merge branch 'master' into msq-disallow-top-level-union-all

.github/workflows/reusable-revised-its.yml

@@ -137,6 +137,7 @@ jobs:
       - name: Collect docker logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
         run: |
+          mkdir docker-logs
           for c in $(docker ps -a --format="{{.Names}}")
           do
             docker logs $c > ./docker-logs/$c.log
@@ -156,8 +157,7 @@ jobs:
       - name: Collect service logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
         run: |
-          docker cp middlemanager:/shared/logs/ ./service-logs
-          tar cvzf ./service-logs.tgz ./service-logs
+          tar cvzf ./service-logs.tgz ~/shared/logs
 
       - name: Upload Druid service logs to GitHub
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}

.github/workflows/reusable-standard-its.yml

@@ -91,6 +91,7 @@ jobs:
       - name: Collect docker logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
         run: |
+          mkdir docker-logs
           for c in $(docker ps -a --format="{{.Names}}")
           do
             docker logs $c > ./docker-logs/$c.log
@@ -110,8 +111,7 @@ jobs:
       - name: Collect service logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
         run: |
-          docker cp druid-middlemanager:/shared/logs/ ./service-logs
-          tar cvzf ./service-logs.tgz ./service-logs
+          tar cvzf ./service-logs.tgz ~/shared/logs
 
       - name: Upload Druid service logs to GitHub
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}

.github/workflows/standard-its.yml

@@ -159,6 +159,7 @@ jobs:
         with:
           path: ~/.m2/repository
           key: maven-${{ runner.os }}-8-${{ github.sha }}
+          restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}
 
       - name: Maven build
         if: steps.maven-restore.outputs.cache-hit != 'true'

.github/workflows/unit-and-integration-tests-unified.yml

@@ -72,6 +72,7 @@ jobs:
         with:
           path: ~/.m2/repository
           key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }}
+          restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}
 
       - name: Cache targets
         id: target

benchmarks/src/test/java/org/apache/druid/benchmark/FlattenJSONBenchmarkUtilTest.java

@@ -20,6 +20,7 @@
 package org.apache.druid.benchmark;
 
 import org.apache.druid.java.util.common.parsers.Parser;
+import org.apache.druid.utils.JvmUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -64,15 +65,31 @@ public void checkEvent1(Map<String, Object> event)
     Assert.assertEquals("129047958", event.get("e2.ad1[0]").toString());
     Assert.assertEquals("1658972185", event.get("e2.ad1[1]").toString());
     Assert.assertEquals("-997010830", event.get("e2.ad1[2]").toString());
-    Assert.assertEquals("-5.8772014847368817E18", event.get("e3.m1").toString());
+
+    // Java 19 changes some floating point string representation
+    // https://bugs.openjdk.org/browse/JDK-8291475
+    if (JvmUtils.majorVersion() < 19) {
+      Assert.assertEquals("-5.8772014847368817E18", event.get("e3.m1").toString());
+    } else {
+      Assert.assertEquals("-5.877201484736882E18", event.get("e3.m1").toString());
+    }
+
     Assert.assertEquals("0.4375433369079904", event.get("e3.m2").toString());
     Assert.assertEquals("0.8510482953607659", event.get("e3.m3").toString());
     Assert.assertEquals("-2.3832626488759337E18", event.get("e3.m4").toString());
-    Assert.assertEquals("7.9789762132607068E18", event.get("e3.am1[0]").toString());
-    Assert.assertEquals("-7.8634787235005573E18", event.get("e3.am1[1]").toString());
-    Assert.assertEquals("8.7372945568982446E18", event.get("e3.am1[2]").toString());
-    Assert.assertEquals("3.1928124802414899E18", event.get("e3.am1[3]").toString());
-    Assert.assertEquals("-3.9806631713718011E18", event.get("e4.e4.m4").toString());
+    if (JvmUtils.majorVersion() < 19) {
+      Assert.assertEquals("7.9789762132607068E18", event.get("e3.am1[0]").toString());
+      Assert.assertEquals("-7.8634787235005573E18", event.get("e3.am1[1]").toString());
+      Assert.assertEquals("8.7372945568982446E18", event.get("e3.am1[2]").toString());
+      Assert.assertEquals("3.1928124802414899E18", event.get("e3.am1[3]").toString());
+      Assert.assertEquals("-3.9806631713718011E18", event.get("e4.e4.m4").toString());
+    } else {
+      Assert.assertEquals("7.978976213260707E18", event.get("e3.am1[0]").toString());
+      Assert.assertEquals("-7.863478723500557E18", event.get("e3.am1[1]").toString());
+      Assert.assertEquals("8.737294556898245E18", event.get("e3.am1[2]").toString());
+      Assert.assertEquals("3.19281248024149E18", event.get("e3.am1[3]").toString());
+      Assert.assertEquals("-3.980663171371801E18", event.get("e4.e4.m4").toString());
+    }
     Assert.assertEquals("-1915243040", event.get("ae1[0].d1").toString());
     Assert.assertEquals("-2020543641", event.get("ae1[1].d1").toString());
     Assert.assertEquals("1414285347", event.get("ae1[2].e1.d2").toString());

distribution/docker/peon.sh

@@ -149,7 +149,8 @@ then
     mkdir -p ${DRUID_DIRS_TO_CREATE}
 fi
 
-# take the ${TASK_JSON} environment variable and base64 decode, unzip and throw it in ${TASK_DIR}/task.json
-mkdir -p ${TASK_DIR}; echo ${TASK_JSON} | base64 -d | gzip -d > ${TASK_DIR}/task.json;
+# take the ${TASK_JSON} environment variable and base64 decode, unzip and throw it in ${TASK_DIR}/task.json.
+# If TASK_JSON is not set, CliPeon will pull the task.json file from deep storage.
+mkdir -p ${TASK_DIR}; [ -n "$TASK_JSON" ] && echo ${TASK_JSON} | base64 -d | gzip -d > ${TASK_DIR}/task.json;
 
 exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main internal peon $@

docs/configuration/index.md

@@ -2112,16 +2112,19 @@ In addition to the normal cache metrics, the caffeine cache implementation also
 
 Uses memcached as cache backend. This allows all processes to share the same cache.
 
-|Property|Description|Default|
-|--------|-----------|-------|
-|`druid.cache.expiration`|Memcached [expiration time](https://code.google.com/p/memcached/wiki/NewCommands#Standard_Protocol).|2592000 (30 days)|
-|`druid.cache.timeout`|Maximum time in milliseconds to wait for a response from Memcached.|500|
-|`druid.cache.hosts`|Comma separated list of Memcached hosts `<host:port>`.|none|
-|`druid.cache.maxObjectSize`|Maximum object size in bytes for a Memcached object.|52428800 (50 MiB)|
-|`druid.cache.memcachedPrefix`|Key prefix for all keys in Memcached.|druid|
-|`druid.cache.numConnections`|Number of memcached connections to use.|1|
-|`druid.cache.protocol`|Memcached communication protocol. Can be binary or text.|binary|
-|`druid.cache.locator`|Memcached locator. Can be consistent or array_mod.|consistent|
+| Property                                  | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | Default           |
+|-------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|
+| `druid.cache.expiration`                  | Memcached [expiration time](https://code.google.com/p/memcached/wiki/NewCommands#Standard_Protocol).                                                                                                                                                                                                                                                                                                                                                                            | 2592000 (30 days) |
+| `druid.cache.timeout`                     | Maximum time in milliseconds to wait for a response from Memcached.                                                                                                                                                                                                                                                                                                                                                                                                             | 500               |
+| `druid.cache.hosts`                       | Comma separated list of Memcached hosts `<host:port>`. Need to specify all nodes when `druid.cache.clientMode` is set to static. Dynamic mode [automatically identifies nodes in your cluster](https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.html) so just specifying the configuration endpoint and port is fine.                                                                                                                                  | none              |
+| `druid.cache.maxObjectSize`               | Maximum object size in bytes for a Memcached object.                                                                                                                                                                                                                                                                                                                                                                                                                            | 52428800 (50 MiB) |
+| `druid.cache.memcachedPrefix`             | Key prefix for all keys in Memcached.                                                                                                                                                                                                                                                                                                                                                                                                                                           | druid             |
+| `druid.cache.numConnections`              | Number of memcached connections to use.                                                                                                                                                                                                                                                                                                                                                                                                                                         | 1                 |
+| `druid.cache.protocol`                    | Memcached communication protocol. Can be binary or text.                                                                                                                                                                                                                                                                                                                                                                                                                        | binary            |
+| `druid.cache.locator`                     | Memcached locator. Can be consistent or array_mod.                                                                                                                                                                                                                                                                                                                                                                                                                              | consistent        |
+| `druid.cache.enableTls`                   | Enable TLS based connection for Memcached client.  Boolean                                                                                                                                                                                                                                                                                                                                                                                                                      | false             |
+| `druid.cache.clientMode`                  | Client Mode. Static mode requires the user to specify individual cluster nodes. Dynamic mode uses [AutoDiscovery](https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.HowAutoDiscoveryWorks.html) feature of AWS Memcached. String. ["static"](https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.Manual.html) or ["dynamic"](https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.Using.ModifyApp.Java.html) | static            |
+| `druid.cache.skipTlsHostnameVerification` | Skip TLS Hostname Verification. Boolean.                                                                                                                                                                                                                                                                                                                                                                                                                                        | true              | 
 
 #### Hybrid
 

docs/design/extensions-contrib/dropwizard.md

@@ -137,6 +137,10 @@ Latest default metrics mapping can be found [here] (https://github.com/apache/dr
     "dimensions": [],
     "type": "gauge"
   },
+  "segment/scan/active": {
+    "dimensions": [],
+    "type": "gauge"
+  },
   "query/segmentAndCache/time": {
     "dimensions": [],
     "type": "timer",

docs/operations/metrics.md

@@ -89,6 +89,7 @@ Most metric values reset each emission period, as specified in `druid.monitoring
 |`query/segment/time`|Milliseconds taken to query individual segment. Includes time to page in the segment from disk.|`id`, `status`, `segment`, `vectorized`.|several hundred milliseconds|
 |`query/wait/time`|Milliseconds spent waiting for a segment to be scanned.|`id`, `segment`|< several hundred milliseconds|
 |`segment/scan/pending`|Number of segments in queue waiting to be scanned.||Close to 0|
+|`segment/scan/active`|Number of segments currently scanned. This metric also indicates how many threads from `druid.processing.numThreads` are currently being used.||Close to `druid.processing.numThreads`|
 |`query/segmentAndCache/time`|Milliseconds taken to query individual segment or hit the cache (if it is enabled on the Historical process).|`id`, `segment`|several hundred milliseconds|
 |`query/cpu/time`|Microseconds of CPU time taken to complete a query.|<p>Common: `dataSource`, `type`, `interval`, `hasFilters`, `duration`, `context`, `remoteAddress`, `id`.</p><p> Aggregation Queries: `numMetrics`, `numComplexMetrics`.</p><p> GroupBy: `numDimensions`.</p><p> TopN: `threshold`, `dimension`.</p>|Varies|
 |`query/count`|Total number of queries.|This metric is only available if the `QueryCountStatsMonitor` module is included.||
@@ -104,6 +105,7 @@ Most metric values reset each emission period, as specified in `druid.monitoring
 |`query/time`|Milliseconds taken to complete a query.|<p>Common: `dataSource`, `type`, `interval`, `hasFilters`, `duration`, `context`, `remoteAddress`, `id`.</p><p> Aggregation Queries: `numMetrics`, `numComplexMetrics`.</p><p> GroupBy: `numDimensions`.</p><p> TopN: `threshold`, `dimension`.</p>|< 1s|
 |`query/wait/time`|Milliseconds spent waiting for a segment to be scanned.|`id`, `segment`|several hundred milliseconds|
 |`segment/scan/pending`|Number of segments in queue waiting to be scanned.||Close to 0|
+|`segment/scan/active`|Number of segments currently scanned. This metric also indicates how many threads from `druid.processing.numThreads` are currently being used.||Close to `druid.processing.numThreads`|
 |`query/cpu/time`|Microseconds of CPU time taken to complete a query.|<p>Common: `dataSource`, `type`, `interval`, `hasFilters`, `duration`, `context`, `remoteAddress`, `id`.</p><p> Aggregation Queries: `numMetrics`, `numComplexMetrics`.</p><p> GroupBy: `numDimensions`. </p><p>TopN: `threshold`, `dimension`.</p>|Varies|
 |`query/count`|Number of total queries.|This metric is only available if the `QueryCountStatsMonitor` module is included.||
 |`query/success/count`|Number of queries successfully processed.|This metric is only available if the `QueryCountStatsMonitor` module is included.||

docs/querying/lookups.md

@@ -62,6 +62,12 @@ SELECT
 FROM sales
 GROUP BY 1
 ```
+The lookup function also accepts the 3rd argument called `replaceMissingValueWith` as a constant string. If your value is missing a lookup for the queried key, the lookup function returns the result value from `replaceMissingValueWith`
+For example:
+```
+LOOKUP(store, 'store_to_country', 'NA')
+```
+If value is missing from `store_to_country` lookup for given key 'store' then it will return `NA`.
 
 They can also be queried using the [JOIN operator](datasource.md#join):
 

docs/querying/math-expr.md

@@ -82,7 +82,7 @@ The following built-in functions are available.
 |concat|concat(expr, expr...) concatenate a list of strings|
 |format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).|
 |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`|
-|lookup|lookup(expr, lookup-name) looks up expr in a registered [query-time lookup](../querying/lookups.md)|
+|lookup|lookup(expr, lookup-name[,replaceMissingValueWith]) looks up expr in a registered,`replaceMissingValueWith` is an optional constant string [query-time lookup](../querying/lookups.md)|
 |parse_long|parse_long(string[, radix]) parses a string as a long with the given radix, or 10 (decimal) if a radix is not provided.|
 |regexp_extract|regexp_extract(expr, pattern[, index]) applies a regular expression pattern and extracts a capture group index, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern. The pattern may match anywhere inside `expr`; if you want to match the entire string instead, use the `^` and `$` markers at the start and end of your pattern.|
 |regexp_like|regexp_like(expr, pattern) returns whether `expr` matches regular expression `pattern`. The pattern may match anywhere inside `expr`; if you want to match the entire string instead, use the `^` and `$` markers at the start and end of your pattern. |

docs/querying/sql-functions.md

@@ -885,7 +885,7 @@ Calculates the base-10 of the numeric expression.
 
 ## LOOKUP
 
-`LOOKUP(<CHARACTER>, <CHARACTER>)`
+`LOOKUP(<CHARACTER>, <CHARACTER>[, <CHARACTER>])`
 
 **Function type:** [Scalar, string](sql-scalar.md#string-functions)
 

docs/querying/sql-scalar.md

@@ -98,7 +98,7 @@ String functions accept strings, and return a type appropriate to the function.
 |`CHAR_LENGTH(expr)`|Alias for `LENGTH`.|
 |`CHARACTER_LENGTH(expr)`|Alias for `LENGTH`.|
 |`STRLEN(expr)`|Alias for `LENGTH`.|
-|`LOOKUP(expr, lookupName)`|Look up `expr` in a registered [query-time lookup table](lookups.md). Note that lookups can also be queried directly using the [`lookup` schema](sql.md#from).|
+|`LOOKUP(expr, lookupName, [replaceMissingValueWith])`|Look up `expr` in a registered [query-time lookup table](lookups.md). Note that lookups can also be queried directly using the [`lookup` schema](sql.md#from). Optional constant replaceMissingValueWith can be passed as 3rd argument to be returned when value is missing from lookup.|
 |`LOWER(expr)`|Returns `expr` in all lowercase.|
 |`UPPER(expr)`|Returns `expr` in all uppercase.|
 |`PARSE_LONG(string, [radix])`|Parses a string into a long (BIGINT) with the given radix, or 10 (decimal) if a radix is not provided.|

extensions-contrib/ambari-metrics-emitter/src/main/resources/defaultWhiteListMap.json

@@ -58,6 +58,7 @@
   "segment/loadQueue/failed": [],
   "segment/loadQueue/size": [],
   "segment/scan/pending": [],
+  "segment/scan/active": [],
   "segment/size": [
     "dataSource"
   ],

extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json

@@ -42,6 +42,10 @@
     "dimensions": [],
     "type": "gauge"
   },
+  "segment/scan/active": {
+    "dimensions": [],
+    "type": "gauge"
+  },
   "query/segmentAndCache/time": {
     "dimensions": [],
     "type": "timer",