6 OpenMetrics specifies the de-facto standard for transmitting cloud-native
7 metrics at scale, with support for both text representation and Protocol
16 - draft-richih-opsawg-openmetrics-00
21 `OpenMetrics <https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md>`_
35 Telemetry module in CSIT currently support only Gauge, Counter and Info.
41 # HELP calls_total Number of calls total
42 # TYPE calls_total counter
43 calls_total{name="api-rx-from-ring",state="active",thread_id="0",thread_lcore="1",thread_name="vpp_main"} 0.0
44 calls_total{name="fib-walk",state="any wait",thread_id="0",thread_lcore="1",thread_name="vpp_main"} 0.0
45 calls_total{name="ip6-mld-process",state="any wait",thread_id="0",thread_lcore="1",thread_name="vpp_main"} 0.0
46 calls_total{name="ip6-ra-process",state="any wait",thread_id="0",thread_lcore="1",thread_name="vpp_main"} 0.0
47 calls_total{name="unix-epoll-input",state="polling",thread_id="0",thread_lcore="1",thread_name="vpp_main"} 39584.0
48 calls_total{name="avf-0/18/6/0-output",state="active",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
49 calls_total{name="avf-0/18/6/0-tx",state="active",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
50 calls_total{name="avf-input",state="polling",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
51 calls_total{name="ethernet-input",state="active",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
52 calls_total{name="ip4-input-no-checksum",state="active",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
53 calls_total{name="ip4-lookup",state="active",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
54 calls_total{name="ip4-rewrite",state="active",thread_id="1",thread_lcore="2",thread_name="vpp_wk_0"} 91.0
55 calls_total{name="avf-0/18/2/0-output",state="active",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
56 calls_total{name="avf-0/18/2/0-tx",state="active",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
57 calls_total{name="avf-input",state="polling",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
58 calls_total{name="ethernet-input",state="active",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
59 calls_total{name="ip4-input-no-checksum",state="active",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
60 calls_total{name="ip4-lookup",state="active",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
61 calls_total{name="ip4-rewrite",state="active",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 91.0
62 calls_total{name="unix-epoll-input",state="polling",thread_id="2",thread_lcore="0",thread_name="vpp_wk_1"} 1.0
65 Anatomy of existing CSIT telemetry implementation
66 -------------------------------------------------
68 Existing implementation consists of several measurment building blocks:
69 the main measuring block running search algorithms (MLR, PLR, SOAK, MRR, ...),
70 the latency measuring block and the several telemetry blocks with or without
71 traffic running on a background.
73 The main measuring block must not be interrupted by any read operation that can
74 impact data plane traffic processing during throughput search algorithm. Thus
75 operational reads are done before (pre-stat) and after (post-stat) that block.
77 Some operational reads must be done while traffic is running and usually
78 consists of two reads (pre-run-stat, post-run-stat) with defined delay between
85 traffic_start(r=mrr) traffic_stop |< measure >|
87 | pre_run_stat post_run_stat | pre_stat | | post_stat
89 --o--------o---------------o---------o-------o--------+-------------------+------o------------>
97 - bash-perf-stat // if extended_debug == True
100 - vpp-enable-packettrace // if extended_debug == True
104 - vpp-show-packettrace // if extended_debug == True
112 |< traffic_trial0 >|< traffic_trial1 >|< traffic_trialN >|
113 | (i=0,t=duration) | (i=1,t=duration) | (i=N,t=duration) |
115 --o------------------------o------------------------o------------------------o--->
123 |< measure >| traffic_start(r=pdr) traffic_stop traffic_start(r=ndr) traffic_stop |< [ latency ] >|
124 | (r=mlr) | | | | | | .9/.5/.1/.0 |
125 | | | pre_run_stat post_run_stat | | pre_run_stat post_run_stat | | |
126 | | | | | | | | | | | |
127 --+-------------------+----o--------o---------------o---------o--------------o--------o---------------o---------o------------[---------------------]--->
135 - bash-perf-stat // if extended_debug == True
138 - vpp-enable-packettrace // if extended_debug == True
142 - vpp-show-packettrace // if extended_debug == True
147 Improving existing solution
148 ---------------------------
150 Improving existing CSIT telemetry implementaion including these areas.
152 - telemetry optimization
153 - reducing ssh overhead
154 - removing stats without added value
155 - telemetry scheduling
157 - improve configuration
161 Exesting stats implementation was abstracted to having pre-/post-run-stats
162 phases. Improvement will be done by merging pre-/post- logic implementation into
163 separated stat-runtime block configurable and locally executed on SUT.
165 This will increase precision, remove complexity and move implementation into
168 OpenMetric format for cloud native metric capturing will be used to ensure
169 integration with post processing module.
175 traffic_start(r=mrr) traffic_stop |< measure >|
177 | |< stat_runtime >| | stat_pre_trial | | stat_post_trial
179 ----o---+--------------------------+---o-------------o------------+-------------------+-----o------------->
187 - vpp-enable-packettrace // if extended_debug == True
190 - vpp-show-packettrace // if extended_debug == True
197 |< traffic_trial0 >|< traffic_trial1 >|< traffic_trialN >|
198 | (i=0,t=duration) | (i=1,t=duration) | (i=N,t=duration) |
200 --o------------------------o------------------------o------------------------o--->
207 |< program0 >|< program1 >|< programN >|
208 | (@=params) | (@=params) | (@=params) |
210 --o------------------------o------------------------o------------------------o--->
219 |< measure >| traffic_start(r=pdr) traffic_stop traffic_start(r=ndr) traffic_stop |< [ latency ] >|
220 | (r=mlr) | | | | | | .9/.5/.1/.0 |
221 | | | |< stat_runtime >| | | |< stat_runtime >| | | |
222 | | | | | | | | | | | |
223 --+-------------------+-----o---+--------------------------+---o--------------o---+--------------------------+---o-----------[---------------------]--->
231 - vpp-enable-packettrace // if extended_debug == True
234 - vpp-show-packettrace // if extended_debug == True
246 - libclang1-9 libllvm9
249 $ sudo apt install bpfcc-tools python3-bpfcc libbpfcc libbpfcc-dev libclang1-9 libllvm9
261 format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
263 format: '%(message)s'
266 class: logging.StreamHandler
269 stream: ext://sys.stdout
271 class: logging.handlers.RotatingFileHandler
274 filename: /tmp/metric.prom
291 documentation: Cycles processed by CPUs
297 - name: cpu_instruction
298 documentation: Instructions retired by CPUs
304 - name: llc_reference
305 documentation: Last level cache operations by type
312 documentation: Last level cache operations by type
319 - type: 0x0 # HARDWARE
320 name: 0x0 # PERF_COUNT_HW_CPU_CYCLES
323 - type: 0x0 # HARDWARE
324 name: 0x1 # PERF_COUNT_HW_INSTRUCTIONS
325 target: on_cpu_instruction
326 table: cpu_instruction
327 - type: 0x0 # HARDWARE
328 name: 0x2 # PERF_COUNT_HW_CACHE_REFERENCES
329 target: on_cache_reference
331 - type: 0x0 # HARDWARE
332 name: 0x3 # PERF_COUNT_HW_CACHE_MISSES
333 target: on_cache_miss
336 #include <linux/ptrace.h>
337 #include <uapi/linux/bpf_perf_event.h>
339 const int max_cpus = 256;
344 char name[TASK_COMM_LEN];
347 BPF_HASH(llc_miss, struct key_t);
348 BPF_HASH(llc_reference, struct key_t);
349 BPF_HASH(cpu_instruction, struct key_t);
350 BPF_HASH(cpu_cycle, struct key_t);
352 static inline __attribute__((always_inline)) void get_key(struct key_t* key) {
353 key->cpu = bpf_get_smp_processor_id();
354 key->pid = bpf_get_current_pid_tgid();
355 bpf_get_current_comm(&(key->name), sizeof(key->name));
358 int on_cpu_cycle(struct bpf_perf_event_data *ctx) {
359 struct key_t key = {};
362 cpu_cycle.increment(key, ctx->sample_period);
365 int on_cpu_instruction(struct bpf_perf_event_data *ctx) {
366 struct key_t key = {};
369 cpu_instruction.increment(key, ctx->sample_period);
372 int on_cache_reference(struct bpf_perf_event_data *ctx) {
373 struct key_t key = {};
376 llc_reference.increment(key, ctx->sample_period);
379 int on_cache_miss(struct bpf_perf_event_data *ctx) {
380 struct key_t key = {};
383 llc_miss.increment(key, ctx->sample_period);
388 CSIT captured metrics
389 ---------------------
398 - BPF_HASH(llc_miss, struct key_t);
399 - BPF_HASH(llc_reference, struct key_t);
400 - BPF_HASH(cpu_instruction, struct key_t);
401 - BPF_HASH(cpu_cycle, struct key_t);
421 - runtime /node `show runtime`
428 - inst-and-clock node intel-core instructions/packet, cycles/packet and IPC
429 - cache-hierarchy node intel-core cache hits and misses
430 - context-switches thread linux per-thread context switches
431 - branch-mispred node intel-core Branches, branches taken and mis-predictions
432 - page-faults thread linux per-thread page faults
433 - load-blocks node intel-core load operations blocked due to various uarch reasons
434 - power-licensing node intel-core Thread power licensing
435 - memory-bandwidth system intel-uncore memory reads and writes per memory controller channel
437 Memory resource - tbd
438 _____________________
440 - memory /segment `show memory verbose api-segment stats-segment main-heap`
447 - max-total-allocated
448 - physmem `show physmem`
455 - counters /node `show node counters`
458 - hardware /interface `show interface`
461 - packets /interface `show hardware`
476 DUT DPDK metrics - tbd
477 ~~~~~~~~~~~~~~~~~~~~~~
483 - BPF_HASH(llc_miss, struct key_t);
484 - BPF_HASH(llc_reference, struct key_t);
485 - BPF_HASH(cpu_instruction, struct key_t);
486 - BPF_HASH(cpu_cycle, struct key_t);