Merge pull request #116 from kubernetes-sigs/scheduler-perf-run

fix: repear scheduler_perf to run correctly

Makefile

@@ -10,7 +10,8 @@ examples/advanced/main.wasm: examples/advanced/main.go
 
 .PHONY: build-tinygo
 build-tinygo: examples/nodenumber/main.wasm examples/advanced/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm \
- guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm guest/testdata/permit/main.wasm
+ guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm guest/testdata/permit/main.wasm \
+ internal/e2e/scheduler_perf/wasm/nodenumber/main.wasm 
 
 %/main-debug.wasm: %/main.go
  @(cd $(@D); tinygo build -o main-debug.wasm -gc=custom -tags=custommalloc -scheduler=none -target=wasi .)

internal/e2e/go.mod

@@ -16,6 +16,7 @@ require (
  k8s.io/klog/v2 v2.90.1
  k8s.io/kube-scheduler v0.27.3
  k8s.io/kubernetes v1.27.3
+ sigs.k8s.io/kube-scheduler-wasm-extension/guest v0.0.0-00010101000000-000000000000
  sigs.k8s.io/kube-scheduler-wasm-extension/scheduler v0.0.0-00010101000000-000000000000
  sigs.k8s.io/yaml v1.3.0
 )
@@ -191,11 +192,12 @@ require (
  k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
  sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
  sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+ sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto v0.0.0-00010101000000-000000000000 // indirect
  sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
 
 replace (
+ sigs.k8s.io/kube-scheduler-wasm-extension/guest => ../../guest
  sigs.k8s.io/kube-scheduler-wasm-extension/kubernetes/proto => ../../kubernetes/proto
-
  sigs.k8s.io/kube-scheduler-wasm-extension/scheduler => ../../scheduler
 )

internal/e2e/scheduler_perf/config/performance-config.yaml

@@ -85,4 +85,4 @@
  params:
  initNodes: 500
  initPods: 500
- measurePods: 1000
+ measurePods: 1000

internal/e2e/scheduler_perf/config/sched-config-wasm-only.yaml

@@ -8,4 +8,4 @@ profiles:
  pluginConfig:
  - name: wasm
  args:
- guestURL: "file://../../../examples/nodenumber/main.wasm"
+ guestURL: "file://./wasm/nodenumber/main.wasm"

internal/e2e/scheduler_perf/wasm/nodenumber/README.md

@@ -0,0 +1,4 @@
+# NodeNumber Plugin
+
+This is the nodenumber example wasm plugin, which only implements PreScore and Score.
+It doesn't use any additional host functions (klog, handle, etc) so that scheduler_perf can measure the overhead truely.

internal/e2e/scheduler_perf/wasm/nodenumber/main.go

@@ -0,0 +1,136 @@
+/*
+ Copyright 2023 The Kubernetes Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+// Package main is the entrypoint of the %.wasm file, compiled with
+// '-target=wasi'. See /guest/RATIONALE.md for details.
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
+ "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto"
+ "sigs.k8s.io/kube-scheduler-wasm-extension/guest/config"
+ "sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog"
+ klogapi "sigs.k8s.io/kube-scheduler-wasm-extension/guest/klog/api"
+ "sigs.k8s.io/kube-scheduler-wasm-extension/guest/plugin"
+)
+
+// main is compiled to a WebAssembly function named "_start", called by the
+// wasm scheduler plugin during initialization.
+func main() {
+ p, err := New(klog.Get(), config.Get())
+ if err != nil {
+ panic(err)
+ }
+ plugin.Set(p)
+}
+
+func New(klog klogapi.Klog, jsonConfig []byte) (api.Plugin, error) {
+ var args nodeNumberArgs
+ if jsonConfig != nil {
+ if err := json.Unmarshal(jsonConfig, &args); err != nil {
+ panic(fmt.Errorf("decode arg into NodeNumberArgs: %w", err))
+ }
+ klog.Info("NodeNumberArgs is successfully applied")
+ }
+ return &NodeNumber{reverse: args.Reverse}, nil
+}
+
+// NodeNumber is an example plugin that favors nodes that share a numerical
+// suffix with the pod name.
+//
+// For example, when a pod named "Pod1" is scheduled, a node named "Node1" gets
+// a higher score than a node named "Node9".
+//
+// # Notes
+//
+// - Only the last character in names are considered. This means "Node99" is
+// treated the same as "Node9"
+// - The reverse field inverts the score. For example, when `reverse == true`
+// a numeric match gets a results in a lower score than a match.
+type NodeNumber struct {
+ reverse bool
+}
+
+type nodeNumberArgs struct {
+ Reverse bool `json:"reverse"`
+}
+
+const (
+ // Name is the name of the plugin used in the plugin registry and configurations.
+ Name = "NodeNumber"
+ preScoreStateKey = "PreScore" + Name
+)
+
+// preScoreState computed at PreScore and used at Score.
+type preScoreState struct {
+ podSuffixNumber uint8
+}
+
+// EventsToRegister implements api.EnqueueExtensions
+func (pl *NodeNumber) EventsToRegister() []api.ClusterEvent {
+ return []api.ClusterEvent{
+ {Resource: api.Node, ActionType: api.Add},
+ }
+}
+
+// PreScore implements api.PreScorePlugin
+func (pl *NodeNumber) PreScore(state api.CycleState, pod proto.Pod, _ proto.NodeList) *api.Status {
+ podnum, ok := lastNumber(pod.Spec().GetNodeName())
+ if !ok {
+ return nil // return success even if its suffix is non-number.
+ }
+ state.Write(preScoreStateKey, &preScoreState{podSuffixNumber: podnum})
+ return nil
+}
+
+// Score implements api.ScorePlugin
+func (pl *NodeNumber) Score(state api.CycleState, pod proto.Pod, nodeName string) (int32, *api.Status) {
+ var match bool
+ if data, ok := state.Read(preScoreStateKey); ok {
+ // Match is when there is a last digit, and it is the pod suffix.
+ nodenum, ok := lastNumber(nodeName)
+ match = ok && data.(*preScoreState).podSuffixNumber == nodenum
+ } else {
+ // Match is also when there is no pod spec node name.
+ match = true
+ }
+
+ if pl.reverse {
+ match = !match // invert the condition.
+ }
+
+ if match {
+ return 10, nil
+ }
+ return 0, nil
+}
+
+// lastNumber returns the last number in the string or false.
+func lastNumber(str string) (uint8, bool) {
+ if len(str) == 0 {
+ return 0, false
+ }
+
+ // We have at least a single character name. See if the last is a digit.
+ lastChar := str[len(str)-1]
+ if '0' <= lastChar && lastChar <= '9' {
+ return lastChar - '0', true
+ }
+ return 0, false
+}

scheduler/plugin/guest.go

@@ -220,6 +220,11 @@ func (g *guest) normalizeScore(ctx context.Context) (framework.NodeScoreList, *f
  statusCode := int32(callStack[0])
  statusReason := paramsFromContext(ctx).resultStatusReason
  normalizedScoreList := paramsFromContext(ctx).resultNormalizedScoreList
+ if len(normalizedScoreList) == 0 {
+ // Probably the guest didn't implement NormalizeScore().
+ normalizedScoreList = paramsFromContext(ctx).nodeScoreList
+ }
+
  return normalizedScoreList, framework.NewStatus(framework.Code(statusCode), statusReason)
 }
 

scheduler/plugin/plugin_test.go

@@ -1380,8 +1380,8 @@ func TestPostBind(t *testing.T) {
  
  wasm error: unreachable
  wasm stack trace:
- .runtime._panic(i32,i32)
- .postbind()
+ main.runtime._panic(i32,i32)
+ main.postbind()
  >`,
  },
  {