8 "git.fd.io/govpp.git/adapter"
9 "git.fd.io/govpp.git/api"
14 RetryUpdateDelay = time.Millisecond * 10
15 HealthCheckInterval = time.Second // default health check probe interval
19 SystemStatsPrefix = "/sys/"
20 SystemStats_VectorRate = SystemStatsPrefix + "vector_rate"
21 SystemStats_NumWorkerThreads = SystemStatsPrefix + "num_worker_threads"
22 SystemStats_VectorRatePerWorker = SystemStatsPrefix + "vector_rate_per_worker"
23 SystemStats_InputRate = SystemStatsPrefix + "input_rate"
24 SystemStats_LastUpdate = SystemStatsPrefix + "last_update"
25 SystemStats_LastStatsClear = SystemStatsPrefix + "last_stats_clear"
26 SystemStats_Heartbeat = SystemStatsPrefix + "heartbeat"
28 NodeStatsPrefix = "/sys/node/"
29 NodeStats_Names = NodeStatsPrefix + "names"
30 NodeStats_Clocks = NodeStatsPrefix + "clocks"
31 NodeStats_Vectors = NodeStatsPrefix + "vectors"
32 NodeStats_Calls = NodeStatsPrefix + "calls"
33 NodeStats_Suspends = NodeStatsPrefix + "suspends"
35 BufferStatsPrefix = "/buffer-pools/"
36 BufferStats_Cached = "cached"
37 BufferStats_Used = "used"
38 BufferStats_Available = "available"
40 CounterStatsPrefix = "/err/"
42 MemoryStatSegPrefix = "/mem/statseg"
43 MemoryStatPrefix = "/mem/stat"
44 MemoryMainPrefix = "/mem/main"
45 MemoryStats_Total = "total"
46 MemoryStats_Used = "used"
48 InterfaceStatsPrefix = "/if/"
49 InterfaceStats_Names = InterfaceStatsPrefix + "names"
50 InterfaceStats_Drops = InterfaceStatsPrefix + "drops"
51 InterfaceStats_Punt = InterfaceStatsPrefix + "punt"
52 InterfaceStats_IP4 = InterfaceStatsPrefix + "ip4"
53 InterfaceStats_IP6 = InterfaceStatsPrefix + "ip6"
54 InterfaceStats_RxNoBuf = InterfaceStatsPrefix + "rx-no-buf"
55 InterfaceStats_RxMiss = InterfaceStatsPrefix + "rx-miss"
56 InterfaceStats_RxError = InterfaceStatsPrefix + "rx-error"
57 InterfaceStats_TxError = InterfaceStatsPrefix + "tx-error"
58 InterfaceStats_Mpls = InterfaceStatsPrefix + "mpls"
59 InterfaceStats_Rx = InterfaceStatsPrefix + "rx"
60 InterfaceStats_RxUnicast = InterfaceStatsPrefix + "rx-unicast"
61 InterfaceStats_RxMulticast = InterfaceStatsPrefix + "rx-multicast"
62 InterfaceStats_RxBroadcast = InterfaceStatsPrefix + "rx-broadcast"
63 InterfaceStats_Tx = InterfaceStatsPrefix + "tx"
64 InterfaceStats_TxUnicast = InterfaceStatsPrefix + "tx-unicast"
65 InterfaceStats_TxUnicastMiss = InterfaceStatsPrefix + "tx-unicast-miss"
66 InterfaceStats_TxMulticast = InterfaceStatsPrefix + "tx-multicast"
67 InterfaceStats_TxBroadcast = InterfaceStatsPrefix + "tx-broadcast"
69 // TODO: network stats
70 NetworkStatsPrefix = "/net/"
71 NetworkStats_RouteTo = NetworkStatsPrefix + "route/to"
72 NetworkStats_RouteVia = NetworkStatsPrefix + "route/via"
73 NetworkStats_MRoute = NetworkStatsPrefix + "mroute"
74 NetworkStats_Adjacency = NetworkStatsPrefix + "adjacency"
75 NetworkStats_Punt = NetworkStatsPrefix + "punt"
78 type StatsConnection struct {
79 statsClient adapter.StatsAPI
81 maxAttempts int // interval for reconnect attempts
82 recInterval time.Duration // maximum number of reconnect attempts
84 connChan chan ConnectionEvent // connection event channel
85 done chan struct{} // to terminate stats connection watcher
87 errorStatsData *adapter.StatDir
88 nodeStatsData *adapter.StatDir
89 ifaceStatsData *adapter.StatDir
90 sysStatsData *adapter.StatDir
91 bufStatsData *adapter.StatDir
92 memStatsData *adapter.StatDir
95 func newStatsConnection(stats adapter.StatsAPI, attempts int, interval time.Duration) *StatsConnection {
97 attempts = DefaultMaxReconnectAttempts
100 interval = DefaultReconnectInterval
103 return &StatsConnection{
105 maxAttempts: attempts,
106 recInterval: interval,
107 connChan: make(chan ConnectionEvent, NotificationChanBufSize),
108 done: make(chan struct{}),
112 // ConnectStats connects to Stats API using specified adapter and returns a connection handle.
113 // This call blocks until it is either connected, or an error occurs.
114 // Only one connection attempt will be performed.
115 func ConnectStats(stats adapter.StatsAPI) (*StatsConnection, error) {
116 log.Debug("Connecting to stats..")
117 c := newStatsConnection(stats, DefaultMaxReconnectAttempts, DefaultReconnectInterval)
119 if err := c.statsClient.Connect(); err != nil {
122 log.Debugf("Connected to stats.")
127 // AsyncConnectStats connects to the VPP stats socket asynchronously and returns the connection
128 // handle with state channel. The call is non-blocking and the caller is expected to watch ConnectionEvent
129 // values from the channel and wait for connect/disconnect events. Connection loop tries to reconnect the
130 // socket in case the session was disconnected.
131 func AsyncConnectStats(stats adapter.StatsAPI, attempts int, interval time.Duration) (*StatsConnection, chan ConnectionEvent, error) {
132 log.Debug("Connecting to stats asynchronously..")
133 c := newStatsConnection(stats, attempts, interval)
137 return c, c.connChan, nil
140 func (c *StatsConnection) connectLoop() {
141 log.Debug("Asynchronously connecting to stats..")
142 var reconnectAttempts int
144 // loop until connected
146 if err := c.statsClient.Connect(); err == nil {
147 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: Connected})
149 } else if reconnectAttempts < c.maxAttempts {
151 log.Warnf("connecting stats failed (attempt %d/%d): %v", reconnectAttempts, c.maxAttempts, err)
152 time.Sleep(c.recInterval)
154 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: Failed, Error: err})
158 // start monitoring stats connection state
162 // Disconnect disconnects from Stats API and releases all connection-related resources.
163 func (c *StatsConnection) Disconnect() {
167 if c.statsClient != nil {
168 if err := c.statsClient.Disconnect(); err != nil {
169 log.Debugf("disconnecting stats client failed: %v", err)
176 func (c *StatsConnection) monitorSocket() {
177 var state, lastState ConnectionState
178 ticker := time.NewTicker(HealthCheckInterval)
183 _, err := c.statsClient.ListStats(SystemStats_Heartbeat)
185 if err == adapter.ErrStatsDataBusy {
186 state = NotResponding
188 if err == adapter.ErrStatsDisconnected {
191 if err == adapter.ErrStatsAccessFailed {
194 if state == lastState {
198 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: state, Error: err})
200 log.Debugf("health check watcher closed")
201 c.sendStatsConnEvent(ConnectionEvent{Timestamp: time.Now(), State: Disconnected, Error: nil})
207 func (c *StatsConnection) updateStats(statDir **adapter.StatDir, patterns ...string) error {
209 panic("statDir must not nil")
211 try := func() error {
212 if (*statDir) == nil {
213 dir, err := c.statsClient.PrepareDir(patterns...)
215 log.Debugln("preparing dir failed:", err)
220 if err := c.statsClient.UpdateDir(*statDir); err != nil {
221 log.Debugln("updating dir failed:", err)
230 for r := 0; r < RetryUpdateCount; r++ {
231 if err = try(); err == nil {
233 log.Debugf("retry successfull (r=%d)", r)
236 } else if err == adapter.ErrStatsDirStale || err == adapter.ErrStatsDataBusy {
239 log.Debugf("sleeping for %v before next try", RetryUpdateDelay)
240 time.Sleep(RetryUpdateDelay)
243 // error is not retryable
250 // GetSystemStats retrieves VPP system stats.
251 func (c *StatsConnection) GetSystemStats(sysStats *api.SystemStats) (err error) {
252 if err := c.updateStats(&c.sysStatsData, SystemStatsPrefix); err != nil {
256 for _, stat := range c.sysStatsData.Entries {
258 if s, ok := stat.Data.(adapter.ScalarStat); ok {
261 switch string(stat.Name) {
262 case SystemStats_VectorRate:
263 sysStats.VectorRate = val
264 case SystemStats_NumWorkerThreads:
265 sysStats.NumWorkerThreads = val
266 case SystemStats_VectorRatePerWorker:
268 if ss, ok := stat.Data.(adapter.SimpleCounterStat); ok {
269 vals = make([]uint64, len(ss))
274 vals[w] = uint64(ss[w][0])
277 sysStats.VectorRatePerWorker = vals
278 case SystemStats_InputRate:
279 sysStats.InputRate = val
280 case SystemStats_LastUpdate:
281 sysStats.LastUpdate = val
282 case SystemStats_LastStatsClear:
283 sysStats.LastStatsClear = val
284 case SystemStats_Heartbeat:
285 sysStats.Heartbeat = val
292 // GetErrorStats retrieves VPP error stats.
293 func (c *StatsConnection) GetErrorStats(errorStats *api.ErrorStats) (err error) {
294 if err := c.updateStats(&c.errorStatsData, CounterStatsPrefix); err != nil {
298 if errorStats.Errors == nil || len(errorStats.Errors) != len(c.errorStatsData.Entries) {
299 errorStats.Errors = make([]api.ErrorCounter, len(c.errorStatsData.Entries))
300 for i := 0; i < len(c.errorStatsData.Entries); i++ {
301 errorStats.Errors[i].CounterName = string(c.errorStatsData.Entries[i].Name)
305 for i, stat := range c.errorStatsData.Entries {
306 if stat.Type != adapter.ErrorIndex {
309 if errStat, ok := stat.Data.(adapter.ErrorStat); ok {
310 values := make([]uint64, len(errStat))
311 for j, errStatW := range errStat {
312 values[j] = uint64(errStatW)
314 errorStats.Errors[i].Values = values
321 func (c *StatsConnection) GetNodeStats(nodeStats *api.NodeStats) (err error) {
322 if err := c.updateStats(&c.nodeStatsData, NodeStatsPrefix); err != nil {
326 prepNodes := func(l int) {
327 if nodeStats.Nodes == nil || len(nodeStats.Nodes) != l {
328 nodeStats.Nodes = make([]api.NodeCounters, l)
329 for i := 0; i < l; i++ {
330 nodeStats.Nodes[i].NodeIndex = uint32(i)
334 perNode := func(stat adapter.StatEntry, fn func(*api.NodeCounters, uint64)) {
335 if s, ok := stat.Data.(adapter.SimpleCounterStat); ok {
337 for i := range nodeStats.Nodes {
338 val := adapter.ReduceSimpleCounterStatIndex(s, i)
339 fn(&nodeStats.Nodes[i], val)
344 for _, stat := range c.nodeStatsData.Entries {
345 switch string(stat.Name) {
346 case NodeStats_Names:
347 if stat, ok := stat.Data.(adapter.NameStat); ok {
349 for i, nc := range nodeStats.Nodes {
350 if nc.NodeName != string(stat[i]) {
351 nc.NodeName = string(stat[i])
352 nodeStats.Nodes[i] = nc
356 case NodeStats_Clocks:
357 perNode(stat, func(node *api.NodeCounters, val uint64) {
360 case NodeStats_Vectors:
361 perNode(stat, func(node *api.NodeCounters, val uint64) {
364 case NodeStats_Calls:
365 perNode(stat, func(node *api.NodeCounters, val uint64) {
368 case NodeStats_Suspends:
369 perNode(stat, func(node *api.NodeCounters, val uint64) {
378 // GetInterfaceStats retrieves VPP per interface stats.
379 func (c *StatsConnection) GetInterfaceStats(ifaceStats *api.InterfaceStats) (err error) {
380 if err := c.updateStats(&c.ifaceStatsData, InterfaceStatsPrefix); err != nil {
384 prep := func(l int) {
385 if ifaceStats.Interfaces == nil || len(ifaceStats.Interfaces) != l {
386 ifaceStats.Interfaces = make([]api.InterfaceCounters, l)
387 for i := 0; i < l; i++ {
388 ifaceStats.Interfaces[i].InterfaceIndex = uint32(i)
392 perNode := func(stat adapter.StatEntry, fn func(*api.InterfaceCounters, uint64)) {
393 if s, ok := stat.Data.(adapter.SimpleCounterStat); ok {
395 for i := range ifaceStats.Interfaces {
396 val := adapter.ReduceSimpleCounterStatIndex(s, i)
397 fn(&ifaceStats.Interfaces[i], val)
401 perNodeComb := func(stat adapter.StatEntry, fn func(*api.InterfaceCounters, [2]uint64)) {
402 if s, ok := stat.Data.(adapter.CombinedCounterStat); ok {
404 for i := range ifaceStats.Interfaces {
405 val := adapter.ReduceCombinedCounterStatIndex(s, i)
406 fn(&ifaceStats.Interfaces[i], val)
411 for _, stat := range c.ifaceStatsData.Entries {
412 switch string(stat.Name) {
413 case InterfaceStats_Names:
414 if stat, ok := stat.Data.(adapter.NameStat); ok {
416 for i, nc := range ifaceStats.Interfaces {
417 if nc.InterfaceName != string(stat[i]) {
418 nc.InterfaceName = string(stat[i])
419 ifaceStats.Interfaces[i] = nc
423 case InterfaceStats_Drops:
424 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
427 case InterfaceStats_Punt:
428 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
431 case InterfaceStats_IP4:
432 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
435 case InterfaceStats_IP6:
436 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
439 case InterfaceStats_RxNoBuf:
440 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
443 case InterfaceStats_RxMiss:
444 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
447 case InterfaceStats_RxError:
448 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
451 case InterfaceStats_TxError:
452 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
455 case InterfaceStats_Mpls:
456 perNode(stat, func(iface *api.InterfaceCounters, val uint64) {
459 case InterfaceStats_Rx:
460 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
461 iface.Rx.Packets = val[0]
462 iface.Rx.Bytes = val[1]
464 case InterfaceStats_RxUnicast:
465 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
466 iface.RxUnicast.Packets = val[0]
467 iface.RxUnicast.Bytes = val[1]
469 case InterfaceStats_RxMulticast:
470 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
471 iface.RxMulticast.Packets = val[0]
472 iface.RxMulticast.Bytes = val[1]
474 case InterfaceStats_RxBroadcast:
475 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
476 iface.RxBroadcast.Packets = val[0]
477 iface.RxBroadcast.Bytes = val[1]
479 case InterfaceStats_Tx:
480 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
481 iface.Tx.Packets = val[0]
482 iface.Tx.Bytes = val[1]
484 case InterfaceStats_TxUnicastMiss:
485 // tx-unicast-miss was a spelling mistake in older versions
488 case InterfaceStats_TxUnicast:
489 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
490 iface.TxUnicast.Packets = val[0]
491 iface.TxUnicast.Bytes = val[1]
493 case InterfaceStats_TxMulticast:
494 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
495 iface.TxMulticast.Packets = val[0]
496 iface.TxMulticast.Bytes = val[1]
498 case InterfaceStats_TxBroadcast:
499 perNodeComb(stat, func(iface *api.InterfaceCounters, val [2]uint64) {
500 iface.TxBroadcast.Packets = val[0]
501 iface.TxBroadcast.Bytes = val[1]
509 // GetBufferStats retrieves VPP buffer pools stats.
510 func (c *StatsConnection) GetBufferStats(bufStats *api.BufferStats) (err error) {
511 if err := c.updateStats(&c.bufStatsData, BufferStatsPrefix); err != nil {
515 if bufStats.Buffer == nil {
516 bufStats.Buffer = make(map[string]api.BufferPool)
519 for _, stat := range c.bufStatsData.Entries {
520 d, f := path.Split(string(stat.Name))
521 d = strings.TrimSuffix(d, "/")
523 name := strings.TrimPrefix(d, BufferStatsPrefix)
524 b, ok := bufStats.Buffer[name]
530 s, ok := stat.Data.(adapter.ScalarStat)
535 case BufferStats_Cached:
537 case BufferStats_Used:
539 case BufferStats_Available:
543 bufStats.Buffer[name] = b
549 func (c *StatsConnection) GetMemoryStats(memStats *api.MemoryStats) (err error) {
550 if err := c.updateStats(&c.memStatsData, MemoryStatSegPrefix, MemoryStatPrefix, MemoryMainPrefix); err != nil {
553 convertStats := func(stats []adapter.Counter) api.MemoryCounters {
554 memUsg := make([]adapter.Counter, 7)
556 return api.MemoryCounters{
557 Total: uint64(memUsg[0]), Used: uint64(memUsg[1]), Free: uint64(memUsg[2]), UsedMMap: uint64(memUsg[3]),
558 TotalAlloc: uint64(memUsg[4]), FreeChunks: uint64(memUsg[5]), Releasable: uint64(memUsg[6]),
562 for _, stat := range c.memStatsData.Entries {
563 if strings.Contains(string(stat.Name), MemoryStatSegPrefix) {
564 _, f := path.Split(string(stat.Name))
566 m, ok := stat.Data.(adapter.ScalarStat)
571 case MemoryStats_Total:
573 case MemoryStats_Used:
576 } else if strings.Contains(string(stat.Name), MemoryStatPrefix) {
577 if perHeapStats, ok := stat.Data.(adapter.SimpleCounterStat); ok {
578 if memStats.Stat == nil {
579 memStats.Stat = make(map[int]api.MemoryCounters)
581 for heap, stats := range perHeapStats {
582 memStats.Stat[heap] = convertStats(stats)
585 } else if strings.Contains(string(stat.Name), MemoryMainPrefix) {
586 if perHeapStats, ok := stat.Data.(adapter.SimpleCounterStat); ok {
587 if memStats.Main == nil {
588 memStats.Main = make(map[int]api.MemoryCounters)
590 for heap, stats := range perHeapStats {
591 memStats.Main[heap] = convertStats(stats)
599 func (c *StatsConnection) sendStatsConnEvent(event ConnectionEvent) {
601 case c.connChan <- event:
603 log.Warn("Stats connection state channel is full, discarding value.")