Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add nodeName template var and update doc #6088

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ The parameters can be used to generate a unique URL to the logs using a template
* - ``{{ .logName }}``
- A deployment specific name where to expect the logs to be
* - ``{{ .hostname }}``
- The hostname where the pod is running and logs reside
- The value used to override the hostname the pod uses internally within its own network namespace (i.e., the pod's ``.spec.hostname``)
* - ``{{ .nodeName }}``
- The hostname of the node where the pod is running and logs reside (i.e., the pod's ``.spec.nodeName``)
* - ``{{ .podRFC3339StartTime }}``
- The pod creation time (in RFC3339 format, e.g. "2021-01-01T02:07:14Z", also conforming to ISO 8601)
* - ``{{ .podRFC3339FinishTime }}``
Expand Down
1 change: 1 addition & 0 deletions flyteplugins/go/tasks/logs/logging_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, tas
ExtraTemplateVars: extraLogTemplateVars,
TaskTemplate: taskTemplate,
HostName: pod.Spec.Hostname,
NodeName: pod.Spec.NodeName,
},
)

Expand Down
33 changes: 27 additions & 6 deletions flyteplugins/go/tasks/logs/logging_utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) {
MessageFormat: core.TaskLog_JSON,
Name: "Stackdriver Logs my-Suffix",
},
}, "")
}, "", "")
})

t.Run("StackDriver", func(t *testing.T) {
Expand All @@ -315,11 +315,11 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) {
MessageFormat: core.TaskLog_JSON,
Name: "Stackdriver Logs my-Suffix",
},
}, "")
}, "", "")
})
}

func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string) {
func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string, nodeName string) {
logPlugin, err := InitializeLogPlugins(config)
assert.NoError(tb, err)

Expand All @@ -335,6 +335,7 @@ func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.Ta
},
},
Hostname: hostname,
NodeName: nodeName,
},
Status: v1.PodStatus{
ContainerStatuses: []v1.ContainerStatus{
Expand Down Expand Up @@ -382,7 +383,7 @@ func TestGetLogsForContainerInPod_Templates(t *testing.T) {
MessageFormat: core.TaskLog_JSON,
Name: "Internal my-Suffix",
},
}, "")
}, "", "")
}

func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) {
Expand All @@ -402,7 +403,27 @@ func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) {
MessageFormat: core.TaskLog_JSON,
Name: "StackDriver my-Suffix",
},
}, "my-hostname")
}, "my-hostname", "")
}

func TestGetLogsForContainerInPodTemplates_NodeName(t *testing.T) {
assertTestSucceeded(t, &LogConfig{
Templates: []tasklog.TemplateLogPlugin{
{
DisplayName: "MyLog",
TemplateURIs: []string{
"{{ .nodeName }}/{{ .namespace }}/{{ .podName }}/{{ .containerName }}/{{ .containerId }}",
},
MessageFormat: core.TaskLog_JSON,
},
},
}, nil, []*core.TaskLog{
{
Uri: "ip-1-2-3-4/my-namespace/my-pod/ContainerName/ContainerID",
MessageFormat: core.TaskLog_JSON,
Name: "MyLog my-Suffix",
},
}, "my-hostname", "ip-1-2-3-4")
}

func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) {
Expand Down Expand Up @@ -562,7 +583,7 @@ func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "")
assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "", "")
})
}
}
1 change: 1 addition & 0 deletions flyteplugins/go/tasks/pluginmachinery/tasklog/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type TemplateVar struct {
// log links.
type Input struct {
HostName string
NodeName string
PodName string
Namespace string
ContainerName string
Expand Down
3 changes: 3 additions & 0 deletions flyteplugins/go/tasks/pluginmachinery/tasklog/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type templateRegexes struct {
ContainerName *regexp.Regexp
ContainerID *regexp.Regexp
Hostname *regexp.Regexp
NodeName *regexp.Regexp
PodRFC3339StartTime *regexp.Regexp
PodRFC3339FinishTime *regexp.Regexp
PodUnixStartTime *regexp.Regexp
Expand All @@ -52,6 +53,7 @@ func initDefaultRegexes() templateRegexes {
MustCreateRegex("containerName"),
MustCreateRegex("containerID"),
MustCreateRegex("hostname"),
MustCreateRegex("nodeName"),
MustCreateRegex("podRFC3339StartTime"),
MustCreateRegex("podRFC3339FinishTime"),
MustCreateRegex("podUnixStartTime"),
Expand Down Expand Up @@ -105,6 +107,7 @@ func (input Input) templateVars() []TemplateVar {
TemplateVar{defaultRegexes.ContainerName, input.ContainerName},
TemplateVar{defaultRegexes.ContainerID, containerID},
TemplateVar{defaultRegexes.Hostname, input.HostName},
TemplateVar{defaultRegexes.NodeName, input.NodeName},
)
if input.TaskExecutionID != nil {
taskExecutionIdentifier := input.TaskExecutionID.GetID()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ func Test_Input_templateVars(t *testing.T) {
}
podBase := Input{
HostName: "my-host",
NodeName: "my-node-name",
PodName: "my-pod",
PodUID: "my-pod-uid",
Namespace: "my-namespace",
Expand Down Expand Up @@ -103,6 +104,7 @@ func Test_Input_templateVars(t *testing.T) {
{defaultRegexes.ContainerName, "my-container"},
{defaultRegexes.ContainerID, "containerID"},
{defaultRegexes.Hostname, "my-host"},
{defaultRegexes.NodeName, "my-node-name"},
{defaultRegexes.PodRFC3339StartTime, "1970-01-01T01:02:03+01:00"},
{defaultRegexes.PodRFC3339FinishTime, "1970-01-01T04:25:45+01:00"},
{defaultRegexes.PodUnixStartTime, "123"},
Expand Down Expand Up @@ -139,6 +141,7 @@ func Test_Input_templateVars(t *testing.T) {
{defaultRegexes.ContainerName, ""},
{defaultRegexes.ContainerID, ""},
{defaultRegexes.Hostname, ""},
{defaultRegexes.NodeName, ""},
{defaultRegexes.NodeID, "n0-0-n0"},
{defaultRegexes.GeneratedName, "generated-name"},
{defaultRegexes.TaskRetryAttempt, "1"},
Expand Down Expand Up @@ -327,12 +330,13 @@ func TestTemplateLogPlugin(t *testing.T) {
{
"ddog",
TemplateLogPlugin{
TemplateURIs: []TemplateURI{"https://app.datadoghq.com/logs?event&from_ts={{ .podUnixStartTime }}&live=true&query=pod_name%3A{{ .podName }}&to_ts={{ .podUnixFinishTime }}"},
TemplateURIs: []TemplateURI{"https://app.datadoghq.com/logs?event&from_ts={{ .podUnixStartTime }}&live=true&query=pod_name%3A{{ .podName }}&to_ts={{ .podUnixFinishTime }}&host={{ .nodeName }}"},
MessageFormat: core.TaskLog_JSON,
},
args{
input: Input{
HostName: "my-host",
NodeName: "ip-1-2-3-4",
PodName: "my-pod",
Namespace: "my-namespace",
ContainerName: "my-container",
Expand All @@ -347,7 +351,7 @@ func TestTemplateLogPlugin(t *testing.T) {
Output{
TaskLogs: []*core.TaskLog{
{
Uri: "https://app.datadoghq.com/logs?event&from_ts=123&live=true&query=pod_name%3Amy-pod&to_ts=12345",
Uri: "https://app.datadoghq.com/logs?event&from_ts=123&live=true&query=pod_name%3Amy-pod&to_ts=12345&host=ip-1-2-3-4",
MessageFormat: core.TaskLog_JSON,
Name: "main_logs",
},
Expand Down