8 "git.fd.io/govpp.git/adapter"
9 "git.fd.io/govpp.git/api"
14 RetryUpdateDelay = time.Millisecond * 10
15 HealthCheckInterval = time.Second // default health check probe interval
19 SystemStatsPrefix = "/sys/"
20 SystemStats_VectorRate = SystemStatsPrefix + "vector_rate"
21 SystemStats_NumWorkerThreads = SystemStatsPrefix + "num_worker_threads"
22 SystemStats_VectorRatePerWorker = SystemStatsPrefix + "vector_rate_per_worker"
23 SystemStats_InputRate = SystemStatsPrefix + "input_rate"
24 SystemStats_LastUpdate = SystemStatsPrefix + "last_update"
25 SystemStats_LastStatsClear = SystemStatsPrefix + "last_stats_clear"
26 SystemStats_Heartbeat = SystemStatsPrefix + "heartbeat"
28 NodeStatsPrefix = "/sys/node/"
29 NodeStats_Names = NodeStatsPrefix + "names"
30 NodeStats_Clocks = NodeStatsPrefix + "clocks"
31 NodeStats_Vectors = NodeStatsPrefix + "vectors"
32 NodeStats_Calls = NodeStatsPrefix + "calls"
33 NodeStats_Suspends = NodeStatsPrefix + "suspends"
35 BufferStatsPrefix = "/buffer-pools/"
36 BufferStats_Cached = "cached"
37 BufferStats_Used = "used"
38 BufferStats_Available = "available"
40 CounterStatsPrefix = "/err/"
42 MemoryStatSegPrefix = "/mem/statseg"
43 MemoryStatSegment = "/mem/stat segment"
44 MemoryMainHeap = "/mem/main heap"
45 MemoryStats_Total = "total"
46 MemoryStats_Used = "used"
48 InterfaceStatsPrefix = "/if/"
49 InterfaceStats_Names = InterfaceStatsPrefix + "names"
50 InterfaceStats_Drops = InterfaceStatsPrefix + "drops"
51 InterfaceStats_Punt = InterfaceStatsPrefix + "punt"
52 InterfaceStats_IP4 = InterfaceStatsPrefix + "ip4"
53 InterfaceStats_IP6 = InterfaceStatsPrefix + "ip6"
54 InterfaceStats_RxNoBuf = InterfaceStatsPrefix + "rx-no-buf"
55 InterfaceStats_RxMiss = InterfaceStatsPrefix + "rx-miss"
56 InterfaceStats_RxError = InterfaceStatsPrefix + "rx-error"
57 InterfaceStats_TxError = InterfaceStatsPrefix + "tx-error"
58 InterfaceStats_Mpls = InterfaceStatsPrefix + "mpls"
59 InterfaceStats_Rx = InterfaceStatsPrefix + "rx"
60 InterfaceStats_RxUnicast = InterfaceStatsPrefix + "rx-unicast"
61 InterfaceStats_RxMulticast = InterfaceStatsPrefix + "rx-multicast"
62 InterfaceStats_RxBroadcast = InterfaceStatsPrefix + "rx-broadcast"
63 InterfaceStats_Tx = InterfaceStatsPrefix + "tx"
64 InterfaceStats_TxUnicast = InterfaceStatsPrefix + "tx-unicast"
65 InterfaceStats_TxUnicastMiss = InterfaceStatsPrefix + "tx-unicast-miss"
66 InterfaceStats_TxMulticast = InterfaceStatsPrefix + "tx-multicast"
67 InterfaceStats_TxBroadcast = InterfaceStatsPrefix + "tx-broadcast"
69 // TODO: network stats
70 NetworkStatsPrefix = "/net/"
71 NetworkStats_RouteTo = NetworkStatsPrefix + "route/to"
72 NetworkStats_RouteVia = NetworkStatsPrefix + "route/via"
73 NetworkStats_MRoute = NetworkStatsPrefix + "mroute"
74 NetworkStats_Adjacency = NetworkStatsPrefix + "adjacency"
75 NetworkStats_Punt = NetworkStatsPrefix + "punt"
78 type StatsConnection struct {
79 statsClient adapter.StatsAPI
81 maxAttempts int // interval for reconnect attempts
82 recInterval time.Duration // maximum number of reconnect attempts
84 connChan chan ConnectionEvent // connection event channel
85 done chan struct{} // to terminate stats connection watcher
87 errorStatsData *adapter.StatDir
88 nodeStatsData *adapter.StatDir
89 ifaceStatsData *adapter.StatDir
90 sysStatsData *adapter.StatDir
91 bufStatsData *adapter.StatDir
92 memStatsData *adapter.StatDir
95 func newStatsConnection(stats adapter.StatsAPI, attempts int, interval time.Duration) *StatsConnection {
97 attempts = DefaultMaxReconnectAttempts
100 interval = DefaultReconnectInterval
103 return &StatsConnection{
105 maxAttempts: attempts,
106 recInterval: interval,
107 connChan: make(chan ConnectionEvent, NotificationChanBufSize),
108 done: make(chan struct{}),
112 // ConnectStats connects to Stats API using specified adapter and returns a connection handle.
113 // This call blocks until it is either connected, or an error occurs.
114 // Only one connection attempt will be performed.
115 func ConnectStats(stats adapter.StatsAPI) (*StatsConnection, error) {
116 log.Debug("Connecting to stats..")
117 c := newStatsConnection(stats, DefaultMaxReconnectAttempts, DefaultReconnectInterval)
119 if err := c.statsClient.Connect(); err != nil {
122 log.Debugf("Connected to stats.")
127 // AsyncConnectStats connects to the VPP stats socket asynchronously and returns the connection
128 // handle with state channel. The call is non-blocking and the caller is expected to watch ConnectionEvent
129 // values from the channel and wait for connect/disconnect events. Connection loop tries to reconnect the
130 // socket in case the session was disconnected.
131 func AsyncConnectStats(stats adapter.StatsAPI, attempts int, interval time.Duration) (*StatsConnection, chan ConnectionEvent, error) {
132 log.Debug("Connecting to stats asynchronously..")
133 c := newStatsConnection(stats, attempts, interval)
137 return c, c.connChan, nil
140 func (c *StatsConnection) connectLoop() {
141 log.Debug("Asynchronously connecting to stats..")
142 var reconnectAttempts int
144 // loop until connected
146 if err := c.statsClient.Connect(); err == nil {
147 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: Connected})
149 } else if reconnectAttempts < c.maxAttempts {
151 log.Warnf("connecting stats failed (attempt %d/%d): %v", reconnectAttempts, c.maxAttempts, err)
152 time.Sleep(c.recInterval)
154 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: Failed, Error: err})
158 // start monitoring stats connection state
162 // Disconnect disconnects from Stats API and releases all connection-related resources.
163 func (c *StatsConnection) Disconnect() {
167 if c.statsClient != nil {
168 if err := c.statsClient.Disconnect(); err != nil {
169 log.Debugf("disconnecting stats client failed: %v", err)
176 func (c *StatsConnection) monitorSocket() {
177 var state, lastState ConnectionState
178 ticker := time.NewTicker(HealthCheckInterval)
183 _, err := c.statsClient.ListStats(SystemStats_Heartbeat)
185 if err == adapter.ErrStatsDataBusy {
186 state = NotResponding
188 if err == adapter.ErrStatsDisconnected {
191 if err == adapter.ErrStatsAccessFailed {
194 if state == lastState {
198 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: state, Error: err})
200 log.Debugf("health check watcher closed")
201 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: Disconnected, Error: nil})
207 func (c *StatsConnection) updateStats(statDir **adapter.StatDir, patterns ...string) error {
209 panic("statDir must not nil")
211 try := func() error {
212 if (*statDir) == nil {
213 dir, err := c.statsClient.PrepareDir(patterns...)
215 log.Debugln("preparing dir failed:", err)
220 if err := c.statsClient.UpdateDir(*statDir); err != nil {
221 log.Debugln("updating dir failed:", err)
230 for r := 0; r < RetryUpdateCount; r++ {
231 if err = try(); err == nil {
233 log.Debugf("retry successfull (r=%d)", r)
236 } else if err == adapter.ErrStatsDirStale || err == adapter.ErrStatsDataBusy {
239 log.Debugf("sleeping for %v before next try", RetryUpdateDelay)
240 time.Sleep(RetryUpdateDelay)
243 // error is not retryable
250 // GetSystemStats retrieves VPP system stats.
251 func (c *StatsConnection) GetSystemStats(sysStats *api.SystemStats) (err error) {
252 if err := c.updateStats(&c.sysStatsData, SystemStatsPrefix); err != nil {
256 for _, stat := range c.sysStatsData.Entries {
258 if s, ok := stat.Data.(adapter.ScalarStat); ok {
261 switch string(stat.Name) {
262 case SystemStats_VectorRate:
263 sysStats.VectorRate = val
264 case SystemStats_NumWorkerThreads:
265 sysStats.NumWorkerThreads = val
266 case SystemStats_VectorRatePerWorker:
268 if ss, ok := stat.Data.(adapter.SimpleCounterStat); ok {
269 vals = make([]uint64, len(ss))
274 vals[w] = uint64(ss[w][0])
277 sysStats.VectorRatePerWorker = vals
278 case SystemStats_InputRate:
279 sysStats.InputRate = val
280 case SystemStats_LastUpdate:
281 sysStats.LastUpdate = val
282 case SystemStats_LastStatsClear:
283 sysStats.LastStatsClear = val
284 case SystemStats_Heartbeat:
285 sysStats.Heartbeat = val
292 // GetErrorStats retrieves VPP error stats.
293 func (c *StatsConnection) GetErrorStats(errorStats *api.ErrorStats) (err error) {
294 if err := c.updateStats(&c.errorStatsData, CounterStatsPrefix); err != nil {
298 if errorStats.Errors == nil || len(errorStats.Errors) != len(c.errorStatsData.Entries) {
299 errorStats.Errors = make([]api.ErrorCounter, len(c.errorStatsData.Entries))
300 for i := 0; i < len(c.errorStatsData.Entries); i++ {
301 errorStats.Errors[i].CounterName = string(c.errorStatsData.Entries[i].Name)
305 for i, stat := range c.errorStatsData.Entries {
306 if errStat, ok := stat.Data.(adapter.ErrorStat); ok {
307 values := make([]uint64, len(errStat))
308 for j, errStatW := range errStat {
309 values[j] = uint64(errStatW)
311 errorStats.Errors[i].Values = values
313 if errStat, ok := stat.Data.(adapter.SimpleCounterStat); ok {
314 values := make([]uint64, len(errStat))
315 for j, errStatW := range errStat {
316 for _, val := range errStatW {
317 values[j] += uint64(val)
320 errorStats.Errors[i].Values = values
326 func (c *StatsConnection) GetNodeStats(nodeStats *api.NodeStats) (err error) {
327 if err := c.updateStats(&c.nodeStatsData, NodeStatsPrefix); err != nil {
331 prepNodes := func(l int) {
332 if nodeStats.Nodes == nil || len(nodeStats.Nodes) != l {
333 nodeStats.Nodes = make([]api.NodeCounters, l)
334 for i := 0; i < l; i++ {
335 nodeStats.Nodes[i].NodeIndex = uint32(i)
339 perNode := func(stat adapter.StatEntry, fn func(*api.NodeCounters, uint64)) {
340 if s, ok := stat.Data.(adapter.SimpleCounterStat); ok {
342 for i := range nodeStats.Nodes {
343 val := adapter.ReduceSimpleCounterStatIndex(s, i)
344 fn(&nodeStats.Nodes[i], val)
349 for _, stat := range c.nodeStatsData.Entries {
350 switch string(stat.Name) {
351 case NodeStats_Names:
352 if stat, ok := stat.Data.(adapter.NameStat); ok {
354 for i, nc := range nodeStats.Nodes {
355 if nc.NodeName != string(stat[i]) {
356 nc.NodeName = string(stat[i])
357 nodeStats.Nodes[i] = nc
361 case NodeStats_Clocks:
362 perNode(stat, func(node *api.NodeCounters, val uint64) {
365 case NodeStats_Vectors:
366 perNode(stat, func(node *api.NodeCounters, val uint64) {
369 case NodeStats_Calls:
370 perNode(stat, func(node *api.NodeCounters, val uint64) {
373 case NodeStats_Suspends:
374 perNode(stat, func(node *api.NodeCounters, val uint64) {
383 // GetInterfaceStats retrieves VPP per interface stats.
384 func (c *StatsConnection) GetInterfaceStats(ifaceStats *api.InterfaceStats) (err error) {
385 if err := c.updateStats(&c.ifaceStatsData, InterfaceStatsPrefix); err != nil {
389 prep := func(l int) {
390 if ifaceStats.Interfaces == nil || len(ifaceStats.Interfaces) != l {
391 ifaceStats.Interfaces = make([]api.InterfaceCounters, l)
392 for i := 0; i < l; i++ {
393 ifaceStats.Interfaces[i].InterfaceIndex = uint32(i)
397 perNode := func(stat adapter.StatEntry, fn func(*api.InterfaceCounters, uint64)) {
398 if s, ok := stat.Data.(adapter.SimpleCounterStat); ok {
400 for i := range ifaceStats.Interfaces {
401 val := adapter.ReduceSimpleCounterStatIndex(s, i)
402 fn(&ifaceStats.Interfaces[i], val)
406 perNodeComb := func(stat adapter.StatEntry, fn func(*api.InterfaceCounters, [2]uint64)) {
407 if s, ok := stat.Data.(adapter.CombinedCounterStat); ok {
409 for i := range ifaceStats.Interfaces {
410 val := adapter.ReduceCombinedCounterStatIndex(s, i)
411 fn(&ifaceStats.Interfaces[i], val)
416 for _, stat := range c.ifaceStatsData.Entries {
417 switch string(stat.Name) {
418 case InterfaceStats_Names:
419 if stat, ok := stat.Data.(adapter.NameStat); ok {
421 for i, nc := range ifaceStats.Interfaces {
422 if nc.InterfaceName != string(stat[i]) {
423 nc.InterfaceName = string(stat[i])
424 ifaceStats.Interfaces[i] = nc
428 case InterfaceStats_Drops:
429 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
432 case InterfaceStats_Punt:
433 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
436 case InterfaceStats_IP4:
437 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
440 case InterfaceStats_IP6:
441 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
444 case InterfaceStats_RxNoBuf:
445 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
448 case InterfaceStats_RxMiss:
449 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
452 case InterfaceStats_RxError:
453 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
456 case InterfaceStats_TxError:
457 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
460 case InterfaceStats_Mpls:
461 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
464 case InterfaceStats_Rx:
465 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
466 iface.Rx.Packets = val[0]
467 iface.Rx.Bytes = val[1]
469 case InterfaceStats_RxUnicast:
470 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
471 iface.RxUnicast.Packets = val[0]
472 iface.RxUnicast.Bytes = val[1]
474 case InterfaceStats_RxMulticast:
475 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
476 iface.RxMulticast.Packets = val[0]
477 iface.RxMulticast.Bytes = val[1]
479 case InterfaceStats_RxBroadcast:
480 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
481 iface.RxBroadcast.Packets = val[0]
482 iface.RxBroadcast.Bytes = val[1]
484 case InterfaceStats_Tx:
485 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
486 iface.Tx.Packets = val[0]
487 iface.Tx.Bytes = val[1]
489 case InterfaceStats_TxUnicastMiss:
490 // tx-unicast-miss was a spelling mistake in older versions
493 case InterfaceStats_TxUnicast:
494 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
495 iface.TxUnicast.Packets = val[0]
496 iface.TxUnicast.Bytes = val[1]
498 case InterfaceStats_TxMulticast:
499 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
500 iface.TxMulticast.Packets = val[0]
501 iface.TxMulticast.Bytes = val[1]
503 case InterfaceStats_TxBroadcast:
504 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
505 iface.TxBroadcast.Packets = val[0]
506 iface.TxBroadcast.Bytes = val[1]
514 // GetBufferStats retrieves VPP buffer pools stats.
515 func (c *StatsConnection) GetBufferStats(bufStats *api.BufferStats) (err error) {
516 if err := c.updateStats(&c.bufStatsData, BufferStatsPrefix); err != nil {
520 if bufStats.Buffer == nil {
521 bufStats.Buffer = make(map[string]api.BufferPool)
524 for _, stat := range c.bufStatsData.Entries {
525 d, f := path.Split(string(stat.Name))
526 d = strings.TrimSuffix(d, "/")
528 name := strings.TrimPrefix(d, BufferStatsPrefix)
529 b, ok := bufStats.Buffer[name]
535 s, ok := stat.Data.(adapter.ScalarStat)
540 case BufferStats_Cached:
542 case BufferStats_Used:
544 case BufferStats_Available:
548 bufStats.Buffer[name] = b
554 func (c *StatsConnection) GetMemoryStats(memStats *api.MemoryStats) (err error) {
555 if err := c.updateStats(&c.memStatsData, MemoryStatSegPrefix, MemoryStatSegment, MemoryMainHeap); err != nil {
558 convertStats := func(stats []adapter.Counter) api.MemoryCounters {
559 memUsg := make([]adapter.Counter, 7)
561 return api.MemoryCounters{
562 Total: uint64(memUsg[0]), Used: uint64(memUsg[1]), Free: uint64(memUsg[2]), UsedMMap: uint64(memUsg[3]),
563 TotalAlloc: uint64(memUsg[4]), FreeChunks: uint64(memUsg[5]), Releasable: uint64(memUsg[6]),
567 for _, stat := range c.memStatsData.Entries {
568 if strings.Contains(string(stat.Name), MemoryStatSegPrefix) {
569 _, f := path.Split(string(stat.Name))
571 m, ok := stat.Data.(adapter.ScalarStat)
576 case MemoryStats_Total:
578 case MemoryStats_Used:
581 } else if string(stat.Name) == MemoryStatSegment {
582 if perHeapStats, ok := stat.Data.(adapter.SimpleCounterStat); ok {
583 if memStats.Stat == nil {
584 memStats.Stat = make(map[int]api.MemoryCounters)
586 for heap, stats := range perHeapStats {
587 memStats.Stat[heap] = convertStats(stats)
590 } else if string(stat.Name) == MemoryMainHeap {
591 if perHeapStats, ok := stat.Data.(adapter.SimpleCounterStat); ok {
592 if memStats.Main == nil {
593 memStats.Main = make(map[int]api.MemoryCounters)
595 for heap, stats := range perHeapStats {
596 memStats.Main[heap] = convertStats(stats)
604 func (c *StatsConnection) sendStatsConnEvent(event ConnectionEvent) {
606 case c.connChan <- event:
608 log.Warn("Stats connection state channel is full, discarding value.")