Skip to content

Commit

Permalink
Merge pull request #673 from Altinity/issue-672
Browse files Browse the repository at this point in the history 
fixing Complex types in Labels for Logs Details visualization
  • Loading branch information
@Slach
Slach authored Nov 28, 2024
2 parents 232147c + e2c8590 commit 203c266
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 19 deletions.
31 changes: 31 additions & 0 deletions docker/clickhouse/init_schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,37 @@ INSERT INTO default.test_logs(event_time, content, level, id, label, detected_fi
INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+(number*10)) AS event_time, concat('Info Log line ', toString(number)) as content, 'Info' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000);
INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+((500+number)*10)) AS event_time, concat('Unknown Log line ', toString(number)) as content, 'Unknown' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000);

DROP TABLE IF EXISTS default.test_logs_with_complex_labels;
CREATE TABLE default.test_logs_with_complex_labels(
`_raw` String CODEC(ZSTD(1)),
`_time` DateTime64(3, 'Asia/Yekaterinburg') CODEC(ZSTD(1)),
`_map` Map(String, String),
`_db_time` DateTime DEFAULT now() CODEC(ZSTD(1)),
`_time_dec` Float64 DEFAULT toFloat64(_time) CODEC(DoubleDelta, Default),
`cluster_name` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'cluster_name') CODEC(ZSTD(1)),
`host` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'host') CODEC(ZSTD(1)),
`pod_namespace` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'pod_namespace') CODEC(ZSTD(1)),
`pod_name` String DEFAULT JSONExtractString(_raw, 'pod_name') CODEC(ZSTD(1)),
`container_name` String DEFAULT JSONExtractString(_raw, 'container_name') CODEC(ZSTD(1)),
`container_image` String DEFAULT JSONExtractString(_raw, 'container_image') CODEC(ZSTD(1)),
`stream` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'stream') CODEC(ZSTD(1)),
`source` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'source') CODEC(ZSTD(1)),
`sourcetype` String DEFAULT JSONExtractString(_raw, 'source_type') CODEC(ZSTD(1)),
`message` String DEFAULT JSONExtractString(_raw, 'message') CODEC(ZSTD(1)),
`bu` LowCardinality(String) DEFAULT JSON_VALUE(_raw, '$.namespace_labels."business-unit-code"'),
INDEX message_ngram_bf message TYPE ngrambf_v1(4, 1024, 2, 0) GRANULARITY 1,
INDEX pod_name_token_bf pod_name TYPE tokenbf_v1(2048, 4, 0) GRANULARITY 4
)
ENGINE = MergeTree
PARTITION BY toDate(_time)
ORDER BY (cluster_name, bu, pod_namespace, pod_name, container_name, _time);

INSERT INTO default.test_logs_with_complex_labels(_raw, _time, _map)
SELECT '{"cluster_name":"test' || toString(number) || '","host":"test","pod_namespace":"test","pod_name":"test","container_name":"test' || toString(number) || '","container_image":"test","stream":"test","source":"test","source_type":"test","namespace_labels":{"business-unit-code":"test"}}' AS _raw,
now64() - INTERVAL number SECOND _time,
map('map_key' || toString(number),'map_value' ||toString(number)) AS _map
FROM numbers(100);

DROP TABLE IF EXISTS default.test_alerts;
CREATE TABLE IF NOT EXISTS default.test_alerts
(
Expand Down
110 changes: 97 additions & 13 deletions docker/grafana/dashboards/test_logs_support.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,90 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 6,
"id": 39,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/672",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 22,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": true,
"showCommonLabels": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"adHocFilters": [
{
"key": "default.test_logs.level",
"operator": "=",
"value": "Info"
}
],
"adHocValuesQuery": "",
"add_metadata": true,
"contextWindowSize": "100",
"database": "default",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"dateTimeColDataType": "_time",
"dateTimeType": "DATETIME64",
"editorMode": "sql",
"extrapolate": true,
"format": "logs",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"initialized": true,
"interval": "",
"intervalFactor": 1,
"query": "SELECT * FROM $table WHERE $timeFilter",
"rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs_with_complex_labels\n\nWHERE \"_time\" >= toDateTime64(1732352032, 3) AND \"_time\" <= toDateTime64(1732524832, 3)\n",
"refId": "A",
"round": "0s",
"showFormattedSQL": true,
"skip_comments": true,
"table": "test_logs_with_complex_labels"
}
],
"title": "Logs WIth Map as Label",
"type": "logs"
},
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/125 and https://github.com/Altinity/clickhouse-grafana/issues/331",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 11,
"x": 0,
"y": 0
"y": 8
},
"id": 2,
"options": {
Expand All @@ -48,8 +117,19 @@
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"adHocFilters": [
{
"key": "default.test_logs.level",
"operator": "=",
"value": "Info"
}
],
"adHocValuesQuery": "",
"add_metadata": true,
"contextWindowSize": "20",
"database": "default",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
Expand All @@ -60,15 +140,18 @@
"dateTimeColDataType": "event_time",
"dateTimeType": "DATETIME",
"datetimeLoading": false,
"editorMode": "sql",
"extrapolate": true,
"format": "logs",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"initialized": true,
"interval": "",
"intervalFactor": 1,
"query": "SELECT *\nFROM $table\n\nWHERE $timeFilter AND $adhoc\n$conditionalTest(AND content ILIKE ${filter:sqlstring},$filter)",
"rawQuery": "SELECT *\nFROM default.test_logs\n\nWHERE event_time >= toDateTime(1653997717) AND event_time <= toDateTime(1654019317) AND 1\n AND content ILIKE '%Info%line 11%' ",
"rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs\n\nWHERE\n event_time >= toDateTime(1732347051) AND event_time <= toDateTime(1732519851)\n AND (level = 'Info')",
"refId": "A",
"round": "0s",
"showFormattedSQL": true,
"skip_comments": true,
"table": "test_logs",
"tableLoading": false
Expand All @@ -83,11 +166,15 @@
"uid": "P4F4839B759FB0509"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 11,
"x": 11,
"y": 0
"y": 8
},
"id": 3,
"options": {
Expand All @@ -100,6 +187,7 @@
"sortOrder": "Ascending",
"wrapLogMessage": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"builderOptions": {
Expand Down Expand Up @@ -192,11 +280,13 @@
"type": "logs"
}
],
"schemaVersion": 39,
"preload": false,
"schemaVersion": 40,
"tags": [],
"templating": {
"list": [
{
"baseFilters": [],
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
Expand All @@ -208,18 +298,14 @@
"value": "Info"
}
],
"hide": 0,
"name": "adhoc_variable",
"skipUrlSync": false,
"type": "adhoc"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"hide": 0,
"name": "filter",
"options": [
{
Expand All @@ -229,7 +315,6 @@
}
],
"query": "",
"skipUrlSync": false,
"type": "textbox"
}
]
Expand All @@ -238,9 +323,8 @@
"from": "now-2d",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {},
"timezone": "",
"timezone": "utc",
"title": "Test Logs support",
"uid": "VtsMXQl7z",
"version": 1,
Expand Down
17 changes: 12 additions & 5 deletions src/datasource/datasource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ export class CHDataSource
} PRECEDING AND CURRENT ROW) AS timestamp
FROM $table
ORDER BY ${inputTimestampColumn}
) WHERE ${inputTimestampColumn} = '${inputTimestampValue}'`;
) WHERE ${inputTimestampColumn} = ${inputTimestampValue}`;
};

const generateQueryForTimestampForward = (inputTimestampColumn, inputTimestampValue, contextWindowSize) => {
Expand All @@ -199,7 +199,7 @@ export class CHDataSource
} FOLLOWING) AS timestamp
FROM $table
ORDER BY ${inputTimestampColumn}
) WHERE ${inputTimestampColumn} = '${inputTimestampValue}'`;
) WHERE ${inputTimestampColumn} = ${inputTimestampValue}`;
};

const generateRequestForTimestampForward = (timestampField, timestamp, currentRowTimestamp, select) => {
Expand Down Expand Up @@ -237,19 +237,26 @@ export class CHDataSource
const timestampColumn = query?.dateTimeColDataType;

const getLogsTimeBoundaries = async () => {
let formattedDate = String(row.timeEpochMs);
if (formattedDate.length > 10) {
formattedDate = `toDateTime64(${row.timeEpochMs}/1000,3)`;
} else {
formattedDate = `'${row.timeUtc}'`;
}

const boundariesRequest =
options?.direction === LogRowContextQueryDirection.Backward
? generateQueryForTimestampBackward(timestampColumn, row.timeUtc, query?.contextWindowSize)
: generateQueryForTimestampForward(timestampColumn, row.timeUtc, query?.contextWindowSize);
? generateQueryForTimestampBackward(timestampColumn, formattedDate, query?.contextWindowSize)
: generateQueryForTimestampForward(timestampColumn, formattedDate, query?.contextWindowSize);

const { stmt, requestId } = this.createQuery(requestOptions, { ...query, query: boundariesRequest });

const result: any = await this._seriesQuery(stmt, requestId + options?.direction);

return result.data[0];
};

const { timestamp } = await getLogsTimeBoundaries();

const getLogContext = async () => {
const contextDataRequest =
options?.direction === LogRowContextQueryDirection.Backward
Expand Down
28 changes: 27 additions & 1 deletion src/datasource/sql-series/toLogs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,33 @@ export const toLogs = (self: any): DataFrame[] => {
}

if (key === messageField) {
frame.addField({ name: key, type: types[key], labels: labels });
const transformObject = (obj) => {
// Check if the input is an object and not null
if (obj && typeof obj === 'object') {
// Create a new object to store the transformed properties
const result = Array.isArray(obj) ? [] : {};

for (const key in obj) {
if (Object.prototype.hasOwnProperty.call(obj, key)) {
const value = obj[key];

// If the value is an object (and not null), convert it to a string
if (value && typeof value === 'object') {
result[key] = JSON.stringify(value);
} else {
// Otherwise, keep the primitive value as it is
result[key] = value;
}
}
}

return result;
}
// Return the original value if it's not an object
return obj;
}

frame.addField({ name: key, type: types[key], labels: transformObject(labels), config: { filterable: false } });
} else if (!labelFields.includes(key) && types[key].fieldType === FieldType.time) {
frame.addField({ name: key, type: FieldType.time });
} else if (!labelFields.includes(key)) {
Expand Down

0 comments on commit 203c266

Please sign in to comment.