Setting latency/delay for an interface

[hellt]

Hi @florianl
Thank your for your work on making native tc API.

I am trying to set a basic netem delay/latency, and likely do something wrong, because when I apply the netem to an interface, all traffic is stopped over this iface.

This is how I do it:

	// open tc session
	tcnl, err := tc.Open(&tc.Config{
		NetNS: nsFd,
	})
	if err != nil {
		return err
	}

	defer func() {
		if err := tcnl.Close(); err != nil {
			log.Errorf("could not close rtnetlink socket: %v\n", err)
		}
	}()

	err = tcnl.SetOption(netlink.ExtendedAcknowledge, true)
	if err != nil {
		return fmt.Errorf("could not set option ExtendedAcknowledge: %v", err)
	}

	qdisc := tc.Object{
		Msg: tc.Msg{
			Family:  unix.AF_UNSPEC,
			Ifindex: uint32(link.Index),
			Handle:  core.BuildHandle(0x1, 0x0),
			Parent:  tc.HandleRoot,
			Info:    0,
		},
		Attribute: tc.Attribute{
			Kind:  "netem",
			Netem: &tc.Netem{},
		},
	}

	delay64 := delay.Milliseconds() // delay is time.Duration
	qdisc.Attribute.Netem.Latency64 = &delay64

	err = tcnl.Qdisc().Replace(&qdisc)
	if err != nil {
		return err
	}

When I print out the resulting netem on that interface I get:

mgmt0 &{Qopt:{Latency:0 Limit:0 Loss:0 Gap:0 Duplicate:0 Jitter:0} Corr:0xc001004110 DelayDist:<nil> Reorder:0xc001004158 Corrupt:0xc001004188 Rate:0xc0010041c0 Ecn:0xc001004214 Rate64:<nil> Latency64:0xc0010040c0 Jitter64:0xc0010040e0 Slot:<nil>} 

[florianl]

Hi @hellt

Latency64 is not the correct attribute for this use case. Here is an example for tc qdisc add dev tcDev root netem delay 100ms:

	tcTime, err := core.Duration2TcTime(100 * time.Millisecond)
	if err != nil {
		return err
	}
	ticks := core.Time2Tick(tcTime)

	qdisc := tc.Object{
		tc.Msg{
			Family:  unix.AF_UNSPEC,
			Ifindex: uint32(devID.Index),
			Handle:  core.BuildHandle(0x1, 0x0),
			Parent:  tc.HandleRoot,
			Info:    0,
		},
		tc.Attribute{
			Kind: "netem",
			// tc qdisc add dev tcDev root netem delay 100ms
			Netem: &tc.Netem{
				Qopt: tc.NetemQopt{
					Latency: ticks,
				},
			},
		},
	}

[hellt]

I think my lack of tc internal structs stands in the way...
I tried the Qopt as you show above, and I am still seeing that traffic entirely stops flowing from the interface I applied netem to.

For comparison, I used netem/pumba CLI tool to apply delay 100ms to eth1 interface of a container under test and did a diff for spew.Sdump(qdisc):

--- private/go-tc-qdisc.txt     2023-07-29 22:36:20.621745367 +0300
+++ private/netem-pumba-qdisc.txt       2023-07-29 22:36:36.818772616 +0300
@@ -2,7 +2,7 @@
  Msg: (tc.Msg) {
   Family: (uint32) 0,
   Ifindex: (uint32) 5609,
-  Handle: (uint32) 65536,
+  Handle: (uint32) 2149449728,
   Parent: (uint32) 4294967295,
   Info: (uint32) 13
  },
@@ -10,12 +10,12 @@
   Kind: (string) (len=5) "netem",
   EgressBlock: (*uint32)(<nil>),
   IngressBlock: (*uint32)(<nil>),
-  HwOffload: (*uint8)(0xc000584296)(0),
+  HwOffload: (*uint8)(0xc00012a146)(0),
   Chain: (*uint32)(<nil>),
-  Stats: (*tc.Stats)(0xc000588450)({
-   Bytes: (uint64) 0,
-   Packets: (uint32) 0,
-   Drops: (uint32) 129,
+  Stats: (*tc.Stats)(0xc00012e420)({
+   Bytes: (uint64) 98,
+   Packets: (uint32) 1,
+   Drops: (uint32) 0,
    Overlimits: (uint32) 0,
    Bps: (uint32) 0,
    Pps: (uint32) 0,
@@ -23,10 +23,10 @@
    Backlog: (uint32) 0
   }),
   XStats: (*tc.XStats)(<nil>),
-  Stats2: (*tc.Stats2)(0xc00058a120)({
-   Bytes: (uint64) 65556,
+  Stats2: (*tc.Stats2)(0xc0001300c0)({
+   Bytes: (uint64) 420906860564,
    Packets: (uint32) 0,
-   Qlen: (uint32) 0,
+   Qlen: (uint32) 1,
    Backlog: (uint32) 0,
    Drops: (uint32) 196632,
    Requeues: (uint32) 0,
@@ -58,30 +58,30 @@
   Pfifo: (*tc.FifoOpt)(<nil>),
   Bfifo: (*tc.FifoOpt)(<nil>),
   Choke: (*tc.Choke)(<nil>),
-  Netem: (*tc.Netem)(0xc0005aa2a0)({
+  Netem: (*tc.Netem)(0xc00014a150)({
    Qopt: (tc.NetemQopt) {
     Latency: (uint32) 1562500,
-    Limit: (uint32) 0,
+    Limit: (uint32) 1000,
     Loss: (uint32) 0,
     Gap: (uint32) 0,
     Duplicate: (uint32) 0,
-    Jitter: (uint32) 0
+    Jitter: (uint32) 156250
    },
-   Corr: (*tc.NetemCorr)(0xc0005842f0)({
-    Delay: (uint32) 0,
+   Corr: (*tc.NetemCorr)(0xc00012a1d0)({
+    Delay: (uint32) 858993459,
     Loss: (uint32) 0,
     Dup: (uint32) 0
    }),
    DelayDist: (*[]int16)(<nil>),
-   Reorder: (*tc.NetemReorder)(0xc000584338)({
+   Reorder: (*tc.NetemReorder)(0xc00012a218)({
     Probability: (uint32) 0,
     Correlation: (uint32) 0
    }),
-   Corrupt: (*tc.NetemCorrupt)(0xc000584368)({
+   Corrupt: (*tc.NetemCorrupt)(0xc00012a248)({
     Probability: (uint32) 0,
     Correlation: (uint32) 0
    }),
-   Rate: (*tc.NetemRate)(0xc0005843a0)({
+   Rate: (*tc.NetemRate)(0xc00012a280)({
     Rate: (uint32) 0,
     PacketOverhead: (int32) 0,
     CellSize: (int32) 0,
@@ -89,8 +89,8 @@
    }),
    Ecn: (*uint32)(<nil>),
    Rate64: (*uint64)(<nil>),
-   Latency64: (*int64)(0xc0005842a0)(100000000),
-   Jitter64: (*int64)(0xc0005842c0)(0),
+   Latency64: (*int64)(0xc00012a180)(100000000),
+   Jitter64: (*int64)(0xc00012a1a0)(10000000),
    Slot: (*tc.NetemSlot)(<nil>)
   }),
   Plug: (*tc.Plug)(<nil>),

Could it be that the Handle must be set to smth else?

ADD1: I think it is solved with adding some Limit

qdisc.Attribute.Netem.Qopt = tc.NetemQopt{
			Latency: ticks,
+			Limit:   1000,
		}

ADD2: It seems the default Limit=0 prevents packets from being buffered during delay. As doc says - https://man7.org/linux/man-pages/man8/tc-netem.8.html - it is a max number of packets qdisc can hold during delay.

[robshakir]

@hellt -- agreed with your finding here, the docs at https://github.com/torvalds/linux/blob/master/include/uapi/linux/pkt_sched.h can be helpful in this regard.

@florianl - thanks for the excellent library.