feat: improve Prometheus compliance (#1022)

* initial impl Signed-off-by: Ruihang Xia <[email protected]> * minor (useless) refactor Signed-off-by: Ruihang Xia <[email protected]> * retrieve metric name Signed-off-by: Ruihang Xia <[email protected]> * add time index column to group by columns filter out NaN in normalize remove NULL in instant manipulator accept form data as HTTP params correct API URL accept second literal as step param * happy clippy Signed-off-by: Ruihang Xia <[email protected]> * update test result Signed-off-by: Ruihang Xia <[email protected]> --------- Signed-off-by: Ruihang Xia <[email protected]>
GreptimeTeam · Feb 20, 2023 · 68b2319 · 68b2319
1 parent 6e9964a
commit 68b2319
Show file tree

Hide file tree

Showing 9 changed files with 332 additions and 195 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/datanode/src/tests/promql_test.rs b/src/datanode/src/tests/promql_test.rs
@@ -82,16 +82,16 @@ async fn sql_insert_promql_query_ceil() {
         UNIX_EPOCH.checked_add(Duration::from_secs(100)).unwrap(),
         Duration::from_secs(5),
         Duration::from_secs(1),
-        "+---------------------+-------------------------------+----------------------------------+\
-        \n| ts                  | ceil(http_requests_total.cpu) | ceil(http_requests_total.memory) |\
-        \n+---------------------+-------------------------------+----------------------------------+\
-        \n| 1970-01-01T00:00:00 | 67                            | 1024                             |\
-        \n| 1970-01-01T00:00:05 | 67                            | 4096                             |\
-        \n| 1970-01-01T00:00:10 | 100                           | 20480                            |\
-        \n| 1970-01-01T00:00:50 | 12424                         | 1334                             |\
-        \n| 1970-01-01T00:01:20 | 0                             | 2334                             |\
-        \n| 1970-01-01T00:01:40 | 49                            | 3334                             |\
-        \n+---------------------+-------------------------------+----------------------------------+",
+        "+---------------------+-------------------------------+----------------------------------+-------+\
+        \n| ts                  | ceil(http_requests_total.cpu) | ceil(http_requests_total.memory) | host  |\
+        \n+---------------------+-------------------------------+----------------------------------+-------+\
+        \n| 1970-01-01T00:00:00 | 67                            | 1024                             | host1 |\
+        \n| 1970-01-01T00:00:05 | 67                            | 4096                             | host1 |\
+        \n| 1970-01-01T00:00:10 | 100                           | 20480                            | host1 |\
+        \n| 1970-01-01T00:01:20 | 0                             | 2334                             | host1 |\
+        \n| 1970-01-01T00:01:40 | 49                            | 3334                             | host1 |\
+        \n| 1970-01-01T00:00:50 | 12424                         | 1334                             | host1 |\
+        \n+---------------------+-------------------------------+----------------------------------+-------+"
     )
     .await;
 }
@@ -142,13 +142,12 @@ async fn aggregators_simple_sum() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+------------+--------------------------+\
-        \n| group      | SUM(http_requests.value) |\
-        \n+------------+--------------------------+\
-        \n|            |                          |\
-        \n| canary     | 700                      |\
-        \n| production | 300                      |\
-        \n+------------+--------------------------+",
+        "+------------+---------------------+--------------------------+\
+        \n| group      | ts                  | SUM(http_requests.value) |\
+        \n+------------+---------------------+--------------------------+\
+        \n| production | 1970-01-01T00:00:00 | 300                      |\
+        \n| canary     | 1970-01-01T00:00:00 | 700                      |\
+        \n+------------+---------------------+--------------------------+",
     )
     .await;
 }
@@ -167,13 +166,12 @@ async fn aggregators_simple_avg() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+------------+--------------------------+\
-        \n| group      | AVG(http_requests.value) |\
-        \n+------------+--------------------------+\
-        \n|            | 0                        |\
-        \n| production | 150                      |\
-        \n| canary     | 350                      |\
-        \n+------------+--------------------------+",
+        "+------------+---------------------+--------------------------+\
+        \n| group      | ts                  | AVG(http_requests.value) |\
+        \n+------------+---------------------+--------------------------+\
+        \n| production | 1970-01-01T00:00:00 | 150                      |\
+        \n| canary     | 1970-01-01T00:00:00 | 350                      |\
+        \n+------------+---------------------+--------------------------+",
     )
     .await;
 }
@@ -192,13 +190,12 @@ async fn aggregators_simple_count() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+------------+----------------------------+\
-        \n| group      | COUNT(http_requests.value) |\
-        \n+------------+----------------------------+\
-        \n|            | 0                          |\
-        \n| canary     | 2                          |\
-        \n| production | 2                          |\
-        \n+------------+----------------------------+",
+        "+------------+---------------------+----------------------------+\
+        \n| group      | ts                  | COUNT(http_requests.value) |\
+        \n+------------+---------------------+----------------------------+\
+        \n| canary     | 1970-01-01T00:00:00 | 2                          |\
+        \n| production | 1970-01-01T00:00:00 | 2                          |\
+        \n+------------+---------------------+----------------------------+",
     )
     .await;
 }
@@ -217,13 +214,12 @@ async fn aggregators_simple_without() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+------------+------------+--------------------------+\
-        \n| group      | job        | SUM(http_requests.value) |\
-        \n+------------+------------+--------------------------+\
-        \n|            |            |                          |\
-        \n| canary     | api-server | 700                      |\
-        \n| production | api-server | 300                      |\
-        \n+------------+------------+--------------------------+",
+        "+------------+------------+---------------------+--------------------------+\
+        \n| group      | job        | ts                  | SUM(http_requests.value) |\
+        \n+------------+------------+---------------------+--------------------------+\
+        \n| production | api-server | 1970-01-01T00:00:00 | 300                      |\
+        \n| canary     | api-server | 1970-01-01T00:00:00 | 700                      |\
+        \n+------------+------------+---------------------+--------------------------+",
     )
     .await;
 }
@@ -241,11 +237,11 @@ async fn aggregators_empty_by() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+--------------------------+\
-        \n| SUM(http_requests.value) |\
-        \n+--------------------------+\
-        \n| 1000                     |\
-        \n+--------------------------+",
+        "+---------------------+--------------------------+\
+        \n| ts                  | SUM(http_requests.value) |\
+        \n+---------------------+--------------------------+\
+        \n| 1970-01-01T00:00:00 | 1000                     |\
+        \n+---------------------+--------------------------+",
     )
     .await;
 }
@@ -263,11 +259,11 @@ async fn aggregators_no_by_without() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+--------------------------+\
-        \n| SUM(http_requests.value) |\
-        \n+--------------------------+\
-        \n| 1000                     |\
-        \n+--------------------------+",
+        "+---------------------+--------------------------+\
+        \n| ts                  | SUM(http_requests.value) |\
+        \n+---------------------+--------------------------+\
+        \n| 1970-01-01T00:00:00 | 1000                     |\
+        \n+---------------------+--------------------------+",
     )
     .await;
 }
@@ -286,13 +282,12 @@ async fn aggregators_empty_without() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+------------+----------+------------+--------------------------+\
-        \n| group      | instance | job        | SUM(http_requests.value) |\
-        \n+------------+----------+------------+--------------------------+\
-        \n|            |          |            |                          |\
-        \n| production | 0        | api-server | 100                      |\
-        \n| production | 1        | api-server | 200                      |\
-        \n+------------+----------+------------+--------------------------+",
+        "+------------+----------+------------+---------------------+--------------------------+\
+        \n| group      | instance | job        | ts                  | SUM(http_requests.value) |\
+        \n+------------+----------+------------+---------------------+--------------------------+\
+        \n| production | 0        | api-server | 1970-01-01T00:00:00 | 100                      |\
+        \n| production | 1        | api-server | 1970-01-01T00:00:00 | 200                      |\
+        \n+------------+----------+------------+---------------------+--------------------------+",
     )
     .await;
 }
@@ -356,11 +351,12 @@ async fn stddev_by_label() {
         unix_epoch_plus_100s(),
         Duration::from_secs(60),
         Duration::from_secs(0),
-        "+----------+-----------------------------+\
-        \n| instance | STDDEV(http_requests.value) |\
-        \n+----------+-----------------------------+\
-        \n| 0        | 258.19888974716116          |\
-        \n+----------+-----------------------------+",
+        "+----------+---------------------+-----------------------------+\
+        \n| instance | ts                  | STDDEV(http_requests.value) |\
+        \n+----------+---------------------+-----------------------------+\
+        \n| 0        | 1970-01-01T00:00:00 | 258.19888974716116          |\
+        \n| 1        | 1970-01-01T00:00:00 | 258.19888974716116          |\
+        \n+----------+---------------------+-----------------------------+",
     )
     .await;
 }

diff --git a/src/promql/src/extension_plan/instant_manipulate.rs b/src/promql/src/extension_plan/instant_manipulate.rs
@@ -41,8 +41,7 @@ use crate::extension_plan::Millisecond;
 ///
 /// This plan will try to align the input time series, for every timestamp between
 /// `start` and `end` with step `interval`. Find in the `lookback` range if data
-/// is missing at the given timestamp. If data is absent in some timestamp, all columns
-/// except the time index will left blank.
+/// is missing at the given timestamp.
 #[derive(Debug)]
 pub struct InstantManipulate {
     start: Millisecond,
@@ -341,6 +340,17 @@ impl InstantManipulateStream {
         take_indices: Vec<Option<u64>>,
         aligned_ts: Vec<Millisecond>,
     ) -> ArrowResult<RecordBatch> {
+        let aligned_ts = aligned_ts
+            .into_iter()
+            .zip(take_indices.iter())
+            .filter_map(|(ts, i)| i.map(|_| ts))
+            .collect::<Vec<_>>();
+        let take_indices = take_indices
+            .iter()
+            .filter(|i| i.is_some())
+            .copied()
+            .collect::<Vec<_>>();
+
         let indices_array = UInt64Array::from(take_indices);
         let mut arrays = record_batch
             .columns()
@@ -349,7 +359,8 @@ impl InstantManipulateStream {
             .collect::<ArrowResult<Vec<_>>>()?;
         arrays[self.time_index] = Arc::new(TimestampMillisecondArray::from(aligned_ts));
 
-        RecordBatch::try_new(record_batch.schema(), arrays)
+        let result = RecordBatch::try_new(record_batch.schema(), arrays)?;
+        Ok(result)
     }
 }
 
@@ -430,11 +441,8 @@ mod test {
             \n| 1970-01-01T00:01:00 | 1     | foo  |\
             \n| 1970-01-01T00:01:30 | 1     | foo  |\
             \n| 1970-01-01T00:02:00 | 1     | foo  |\
-            \n| 1970-01-01T00:02:30 |       |      |\
             \n| 1970-01-01T00:03:00 | 1     | foo  |\
-            \n| 1970-01-01T00:03:30 |       |      |\
             \n| 1970-01-01T00:04:00 | 1     | foo  |\
-            \n| 1970-01-01T00:04:30 |       |      |\
             \n| 1970-01-01T00:05:00 | 1     | foo  |\
             \n+---------------------+-------+------+",
         );
@@ -449,34 +457,19 @@ mod test {
             \n+---------------------+-------+------+\
             \n| 1970-01-01T00:00:00 | 1     | foo  |\
             \n| 1970-01-01T00:00:10 | 1     | foo  |\
-            \n| 1970-01-01T00:00:20 |       |      |\
             \n| 1970-01-01T00:00:30 | 1     | foo  |\
             \n| 1970-01-01T00:00:40 | 1     | foo  |\
-            \n| 1970-01-01T00:00:50 |       |      |\
             \n| 1970-01-01T00:01:00 | 1     | foo  |\
             \n| 1970-01-01T00:01:10 | 1     | foo  |\
-            \n| 1970-01-01T00:01:20 |       |      |\
             \n| 1970-01-01T00:01:30 | 1     | foo  |\
             \n| 1970-01-01T00:01:40 | 1     | foo  |\
-            \n| 1970-01-01T00:01:50 |       |      |\
             \n| 1970-01-01T00:02:00 | 1     | foo  |\
             \n| 1970-01-01T00:02:10 | 1     | foo  |\
-            \n| 1970-01-01T00:02:20 |       |      |\
-            \n| 1970-01-01T00:02:30 |       |      |\
-            \n| 1970-01-01T00:02:40 |       |      |\
-            \n| 1970-01-01T00:02:50 |       |      |\
             \n| 1970-01-01T00:03:00 | 1     | foo  |\
             \n| 1970-01-01T00:03:10 | 1     | foo  |\
-            \n| 1970-01-01T00:03:20 |       |      |\
-            \n| 1970-01-01T00:03:30 |       |      |\
-            \n| 1970-01-01T00:03:40 |       |      |\
-            \n| 1970-01-01T00:03:50 |       |      |\
             \n| 1970-01-01T00:04:00 | 1     | foo  |\
             \n| 1970-01-01T00:04:10 | 1     | foo  |\
-            \n| 1970-01-01T00:04:20 |       |      |\
-            \n| 1970-01-01T00:04:30 |       |      |\
             \n| 1970-01-01T00:04:40 | 1     | foo  |\
-            \n| 1970-01-01T00:04:50 |       |      |\
             \n| 1970-01-01T00:05:00 | 1     | foo  |\
             \n+---------------------+-------+------+",
         );
@@ -527,14 +520,10 @@ mod test {
             \n| 1970-01-01T00:02:10 | 1     | foo  |\
             \n| 1970-01-01T00:02:20 | 1     | foo  |\
             \n| 1970-01-01T00:02:30 | 1     | foo  |\
-            \n| 1970-01-01T00:02:40 |       |      |\
-            \n| 1970-01-01T00:02:50 |       |      |\
             \n| 1970-01-01T00:03:00 | 1     | foo  |\
             \n| 1970-01-01T00:03:10 | 1     | foo  |\
             \n| 1970-01-01T00:03:20 | 1     | foo  |\
             \n| 1970-01-01T00:03:30 | 1     | foo  |\
-            \n| 1970-01-01T00:03:40 |       |      |\
-            \n| 1970-01-01T00:03:50 |       |      |\
             \n| 1970-01-01T00:04:00 | 1     | foo  |\
             \n| 1970-01-01T00:04:10 | 1     | foo  |\
             \n| 1970-01-01T00:04:20 | 1     | foo  |\
@@ -617,22 +606,8 @@ mod test {
             "+---------------------+-------+------+\
             \n| timestamp           | value | path |\
             \n+---------------------+-------+------+\
-            \n| 1970-01-01T00:03:50 |       |      |\
-            \n| 1970-01-01T00:03:51 |       |      |\
-            \n| 1970-01-01T00:03:52 |       |      |\
-            \n| 1970-01-01T00:03:53 |       |      |\
-            \n| 1970-01-01T00:03:54 |       |      |\
-            \n| 1970-01-01T00:03:55 |       |      |\
-            \n| 1970-01-01T00:03:56 |       |      |\
-            \n| 1970-01-01T00:03:57 |       |      |\
-            \n| 1970-01-01T00:03:58 |       |      |\
-            \n| 1970-01-01T00:03:59 |       |      |\
             \n| 1970-01-01T00:04:00 | 1     | foo  |\
             \n| 1970-01-01T00:04:01 | 1     | foo  |\
-            \n| 1970-01-01T00:04:02 |       |      |\
-            \n| 1970-01-01T00:04:03 |       |      |\
-            \n| 1970-01-01T00:04:04 |       |      |\
-            \n| 1970-01-01T00:04:05 |       |      |\
             \n+---------------------+-------+------+",
         );
         do_normalize_test(230_000, 245_000, 0, 1_000, expected).await;
@@ -646,7 +621,6 @@ mod test {
             \n+---------------------+-------+------+\
             \n| 1970-01-01T00:00:00 | 1     | foo  |\
             \n| 1970-01-01T00:00:10 | 1     | foo  |\
-            \n| 1970-01-01T00:00:20 |       |      |\
             \n| 1970-01-01T00:00:30 | 1     | foo  |\
             \n+---------------------+-------+------+",
         );
@@ -659,37 +633,12 @@ mod test {
             "+---------------------+-------+------+\
             \n| timestamp           | value | path |\
             \n+---------------------+-------+------+\
-            \n| 1969-12-31T23:45:00 |       |      |\
-            \n| 1969-12-31T23:46:00 |       |      |\
-            \n| 1969-12-31T23:47:00 |       |      |\
-            \n| 1969-12-31T23:48:00 |       |      |\
-            \n| 1969-12-31T23:49:00 |       |      |\
-            \n| 1969-12-31T23:50:00 |       |      |\
-            \n| 1969-12-31T23:51:00 |       |      |\
-            \n| 1969-12-31T23:52:00 |       |      |\
-            \n| 1969-12-31T23:53:00 |       |      |\
-            \n| 1969-12-31T23:54:00 |       |      |\
-            \n| 1969-12-31T23:55:00 |       |      |\
-            \n| 1969-12-31T23:56:00 |       |      |\
-            \n| 1969-12-31T23:57:00 |       |      |\
-            \n| 1969-12-31T23:58:00 |       |      |\
-            \n| 1969-12-31T23:59:00 |       |      |\
             \n| 1970-01-01T00:00:00 | 1     | foo  |\
             \n| 1970-01-01T00:01:00 | 1     | foo  |\
             \n| 1970-01-01T00:02:00 | 1     | foo  |\
             \n| 1970-01-01T00:03:00 | 1     | foo  |\
             \n| 1970-01-01T00:04:00 | 1     | foo  |\
             \n| 1970-01-01T00:05:00 | 1     | foo  |\
-            \n| 1970-01-01T00:06:00 |       |      |\
-            \n| 1970-01-01T00:07:00 |       |      |\
-            \n| 1970-01-01T00:08:00 |       |      |\
-            \n| 1970-01-01T00:09:00 |       |      |\
-            \n| 1970-01-01T00:10:00 |       |      |\
-            \n| 1970-01-01T00:11:00 |       |      |\
-            \n| 1970-01-01T00:12:00 |       |      |\
-            \n| 1970-01-01T00:13:00 |       |      |\
-            \n| 1970-01-01T00:14:00 |       |      |\
-            \n| 1970-01-01T00:15:00 |       |      |\
             \n+---------------------+-------+------+",
         );
         do_normalize_test(-900_000, 900_000, 30_000, 60_000, expected).await;
@@ -704,8 +653,6 @@ mod test {
             \n| 1970-01-01T00:03:10 | 1     | foo  |\
             \n| 1970-01-01T00:03:20 | 1     | foo  |\
             \n| 1970-01-01T00:03:30 | 1     | foo  |\
-            \n| 1970-01-01T00:03:40 |       |      |\
-            \n| 1970-01-01T00:03:50 |       |      |\
             \n| 1970-01-01T00:04:00 | 1     | foo  |\
             \n| 1970-01-01T00:04:10 | 1     | foo  |\
             \n| 1970-01-01T00:04:20 | 1     | foo  |\