src/plugins/ioam/export-common/ioam_export.h

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #ifndef __included_ioam_export_h__
  16 #define __included_ioam_export_h__
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/ip/ip.h>
  20 #include <vnet/ip/ip_packet.h>
  21 #include <vnet/ip/ip4_packet.h>
  22 #include <vnet/ip/ip6_packet.h>
  23 #include <vnet/ip/udp.h>
  24 #include <vnet/flow/ipfix_packet.h>
  25
  26 #include <vppinfra/pool.h>
  27 #include <vppinfra/hash.h>
  28 #include <vppinfra/error.h>
  29 #include <vppinfra/elog.h>
  30
  31 #include <vlib/threads.h>
  32
  33 typedef struct ioam_export_buffer
  34 {
  35   /* Allocated buffer */
  36   u32 buffer_index;
  37   u64 touched_at;
  38   u8 records_in_this_buffer;
  39 } ioam_export_buffer_t;
  40
  41
  42 typedef struct
  43 {
  44   /* API message ID base */
  45   u16 msg_id_base;
  46
  47   /* TODO: to support multiple collectors all this has to be grouped and create a vector here */
  48   u8 *record_header;
  49   u32 sequence_number;
  50   u32 domain_id;
  51
  52   /* ipfix collector, our ip address */
  53   ip4_address_t ipfix_collector;
  54   ip4_address_t src_address;
  55
  56   /* Pool of ioam_export_buffer_t */
  57   ioam_export_buffer_t *buffer_pool;
  58   /* Vector of per thread ioam_export_buffer_t to buffer pool index */
  59   u32 *buffer_per_thread;
  60   /* Lock per thread to swap buffers between worker and timer process */
  61   volatile u32 **lockp;
  62
  63   /* time scale transform */
  64   u32 unix_time_0;
  65   f64 vlib_time_0;
  66
  67   /* convenience */
  68   vlib_main_t *vlib_main;
  69   vnet_main_t *vnet_main;
  70   u32 ip4_lookup_node_index;
  71
  72   uword my_hbh_slot;
  73   u32 export_process_node_index;
  74 } ioam_export_main_t;
  75
  76 ioam_export_main_t ioam_export_main;
  77 ioam_export_main_t vxlan_gpe_ioam_export_main;
  78
  79 extern vlib_node_registration_t export_node;
  80 extern vlib_node_registration_t vxlan_export_node;
  81
  82 #define DEFAULT_EXPORT_SIZE (3 * CLIB_CACHE_LINE_BYTES)
  83 /*
  84  *  Number of records in a buffer
  85  * ~(MTU (1500) - [ip hdr(40) + UDP(8) + ipfix (24)]) / DEFAULT_EXPORT_SIZE
  86  */
  87 #define DEFAULT_EXPORT_RECORDS 7
  88
  89 always_inline ioam_export_buffer_t *
  90 ioam_export_get_my_buffer (ioam_export_main_t * em, u32 thread_id)
  91 {
  92
  93   if (vec_len (em->buffer_per_thread) > thread_id)
  94     return (pool_elt_at_index
  95             (em->buffer_pool, em->buffer_per_thread[thread_id]));
  96   return (0);
  97 }
  98
  99 inline static int
 100 ioam_export_buffer_add_header (ioam_export_main_t * em, vlib_buffer_t * b0)
 101 {
 102   clib_memcpy (b0->data, em->record_header, vec_len (em->record_header));
 103   b0->current_data = 0;
 104   b0->current_length = vec_len (em->record_header);
 105   b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 106   return (1);
 107 }
 108
 109 inline static int
 110 ioam_export_init_buffer (ioam_export_main_t * em, vlib_main_t * vm,
 111                          ioam_export_buffer_t * eb)
 112 {
 113   vlib_buffer_t *b = 0;
 114
 115   if (!eb)
 116     return (-1);
 117   /* TODO: Perhaps buffer init from template here */
 118   if (vlib_buffer_alloc (vm, &(eb->buffer_index), 1) != 1)
 119     return (-2);
 120   eb->records_in_this_buffer = 0;
 121   eb->touched_at = vlib_time_now (vm);
 122   b = vlib_get_buffer (vm, eb->buffer_index);
 123   (void) ioam_export_buffer_add_header (em, b);
 124   vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
 125   vnet_buffer (b)->sw_if_index[VLIB_TX] = ~0;
 126   return (1);
 127 }
 128
 129 inline static void
 130 ioam_export_thread_buffer_free (ioam_export_main_t * em)
 131 {
 132   vlib_main_t *vm = em->vlib_main;
 133   ioam_export_buffer_t *eb = 0;
 134   int i;
 135   for (i = 0; i < vec_len (em->buffer_per_thread); i++)
 136     {
 137       eb = pool_elt_at_index (em->buffer_pool, em->buffer_per_thread[i]);
 138       if (eb)
 139         vlib_buffer_free (vm, &(eb->buffer_index), 1);
 140     }
 141   for (i = 0; i < vec_len (em->lockp); i++)
 142     clib_mem_free ((void *) em->lockp[i]);
 143   vec_free (em->buffer_per_thread);
 144   pool_free (em->buffer_pool);
 145   vec_free (em->lockp);
 146   em->buffer_per_thread = 0;
 147   em->buffer_pool = 0;
 148   em->lockp = 0;
 149 }
 150
 151 inline static int
 152 ioam_export_thread_buffer_init (ioam_export_main_t * em, vlib_main_t * vm)
 153 {
 154   int no_of_threads = vec_len (vlib_worker_threads);
 155   int i;
 156   ioam_export_buffer_t *eb = 0;
 157   vlib_node_t *ip4_lookup_node;
 158
 159   pool_alloc_aligned (em->buffer_pool,
 160                       no_of_threads - 1, CLIB_CACHE_LINE_BYTES);
 161   vec_validate_aligned (em->buffer_per_thread,
 162                         no_of_threads - 1, CLIB_CACHE_LINE_BYTES);
 163   vec_validate_aligned (em->lockp, no_of_threads - 1, CLIB_CACHE_LINE_BYTES);
 164   ip4_lookup_node = vlib_get_node_by_name (vm, (u8 *) "ip4-lookup");
 165   em->ip4_lookup_node_index = ip4_lookup_node->index;
 166   if (!em->buffer_per_thread || !em->buffer_pool || !em->lockp)
 167     {
 168       return (-1);
 169     }
 170   for (i = 0; i < no_of_threads; i++)
 171     {
 172       eb = 0;
 173       pool_get_aligned (em->buffer_pool, eb, CLIB_CACHE_LINE_BYTES);
 174       memset (eb, 0, sizeof (*eb));
 175       em->buffer_per_thread[i] = eb - em->buffer_pool;
 176       if (ioam_export_init_buffer (em, vm, eb) != 1)
 177         {
 178           ioam_export_thread_buffer_free (em);
 179           return (-2);
 180         }
 181       em->lockp[i] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
 182                                              CLIB_CACHE_LINE_BYTES);
 183       memset ((void *) em->lockp[i], 0, CLIB_CACHE_LINE_BYTES);
 184     }
 185   return (1);
 186 }
 187
 188 #define IPFIX_IOAM_EXPORT_ID 272
 189
 190 /* Used to build the rewrite */
 191 /* data set packet */
 192 typedef struct
 193 {
 194   ipfix_message_header_t h;
 195   ipfix_set_header_t s;
 196 } ipfix_data_packet_t;
 197
 198 typedef struct
 199 {
 200   ip4_header_t ip4;
 201   udp_header_t udp;
 202   ipfix_data_packet_t ipfix;
 203 } ip4_ipfix_data_packet_t;
 204
 205
 206 inline static void
 207 ioam_export_header_cleanup (ioam_export_main_t * em,
 208                             ip4_address_t * collector_address,
 209                             ip4_address_t * src_address)
 210 {
 211   vec_free (em->record_header);
 212   em->record_header = 0;
 213 }
 214
 215 inline static int
 216 ioam_export_header_create (ioam_export_main_t * em,
 217                            ip4_address_t * collector_address,
 218                            ip4_address_t * src_address)
 219 {
 220   ip4_header_t *ip;
 221   udp_header_t *udp;
 222   ipfix_message_header_t *h;
 223   ipfix_set_header_t *s;
 224   u8 *rewrite = 0;
 225   ip4_ipfix_data_packet_t *tp;
 226
 227
 228   /* allocate rewrite space */
 229   vec_validate_aligned (rewrite,
 230                         sizeof (ip4_ipfix_data_packet_t) - 1,
 231                         CLIB_CACHE_LINE_BYTES);
 232
 233   tp = (ip4_ipfix_data_packet_t *) rewrite;
 234   ip = (ip4_header_t *) & tp->ip4;
 235   udp = (udp_header_t *) (ip + 1);
 236   h = (ipfix_message_header_t *) (udp + 1);
 237   s = (ipfix_set_header_t *) (h + 1);
 238
 239   ip->ip_version_and_header_length = 0x45;
 240   ip->ttl = 254;
 241   ip->protocol = IP_PROTOCOL_UDP;
 242   ip->src_address.as_u32 = src_address->as_u32;
 243   ip->dst_address.as_u32 = collector_address->as_u32;
 244   udp->src_port = clib_host_to_net_u16 (4939 /* $$FIXME */ );
 245   udp->dst_port = clib_host_to_net_u16 (4939);
 246   /* FIXUP: UDP length */
 247   udp->length = clib_host_to_net_u16 (vec_len (rewrite) +
 248                                       (DEFAULT_EXPORT_RECORDS *
 249                                        DEFAULT_EXPORT_SIZE) - sizeof (*ip));
 250
 251   /* FIXUP: message header export_time */
 252   /* FIXUP: message header sequence_number */
 253   h->domain_id = clib_host_to_net_u32 (em->domain_id);
 254
 255   /*FIXUP: Setid length in octets if records exported are not default */
 256   s->set_id_length = ipfix_set_id_length (IPFIX_IOAM_EXPORT_ID,
 257                                           (sizeof (*s) +
 258                                            (DEFAULT_EXPORT_RECORDS *
 259                                             DEFAULT_EXPORT_SIZE)));
 260
 261   /* FIXUP: h version and length length in octets if records exported are not default */
 262   h->version_length = version_length (sizeof (*h) +
 263                                       (sizeof (*s) +
 264                                        (DEFAULT_EXPORT_RECORDS *
 265                                         DEFAULT_EXPORT_SIZE)));
 266
 267   /* FIXUP: ip length if records exported are not default */
 268   /* FIXUP: ip checksum if records exported are not default */
 269   ip->length = clib_host_to_net_u16 (vec_len (rewrite) +
 270                                      (DEFAULT_EXPORT_RECORDS *
 271                                       DEFAULT_EXPORT_SIZE));
 272   ip->checksum = ip4_header_checksum (ip);
 273   _vec_len (rewrite) = sizeof (ip4_ipfix_data_packet_t);
 274   em->record_header = rewrite;
 275   return (1);
 276 }
 277
 278 inline static int
 279 ioam_export_send_buffer (ioam_export_main_t * em, vlib_main_t * vm,
 280                          ioam_export_buffer_t * eb)
 281 {
 282   ip4_header_t *ip;
 283   udp_header_t *udp;
 284   ipfix_message_header_t *h;
 285   ipfix_set_header_t *s;
 286   ip4_ipfix_data_packet_t *tp;
 287   vlib_buffer_t *b0;
 288   u16 new_l0, old_l0;
 289   ip_csum_t sum0;
 290   vlib_frame_t *nf = 0;
 291   u32 *to_next;
 292
 293   b0 = vlib_get_buffer (vm, eb->buffer_index);
 294   tp = vlib_buffer_get_current (b0);
 295   ip = (ip4_header_t *) & tp->ip4;
 296   udp = (udp_header_t *) (ip + 1);
 297   h = (ipfix_message_header_t *) (udp + 1);
 298   s = (ipfix_set_header_t *) (h + 1);
 299
 300   /* FIXUP: message header export_time */
 301   h->export_time = clib_host_to_net_u32 ((u32)
 302                                          (((f64) em->unix_time_0) +
 303                                           (vlib_time_now (em->vlib_main) -
 304                                            em->vlib_time_0)));
 305
 306   /* FIXUP: message header sequence_number */
 307   h->sequence_number = clib_host_to_net_u32 (em->sequence_number++);
 308
 309   /* FIXUP: lengths if different from default */
 310   if (PREDICT_FALSE (eb->records_in_this_buffer != DEFAULT_EXPORT_RECORDS))
 311     {
 312       s->set_id_length =
 313         ipfix_set_id_length (IPFIX_IOAM_EXPORT_ID /* set_id */ ,
 314                              b0->current_length - (sizeof (*ip) +
 315                                                    sizeof (*udp) +
 316                                                    sizeof (*h)));
 317       h->version_length =
 318         version_length (b0->current_length - (sizeof (*ip) + sizeof (*udp)));
 319       sum0 = ip->checksum;
 320       old_l0 = ip->length;
 321       new_l0 = clib_host_to_net_u16 ((u16) b0->current_length);
 322       sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t,
 323                              length /* changed member */ );
 324       ip->checksum = ip_csum_fold (sum0);
 325       ip->length = new_l0;
 326       udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip));
 327     }
 328
 329   /* Enqueue pkts to ip4-lookup */
 330
 331   nf = vlib_get_frame_to_node (vm, em->ip4_lookup_node_index);
 332   nf->n_vectors = 0;
 333   to_next = vlib_frame_vector_args (nf);
 334   nf->n_vectors = 1;
 335   to_next[0] = eb->buffer_index;
 336   vlib_put_frame_to_node (vm, em->ip4_lookup_node_index, nf);
 337   return (1);
 338
 339 }
 340
 341 #define EXPORT_TIMEOUT (20.0)
 342 #define THREAD_PERIOD (30.0)
 343 inline static uword
 344 ioam_export_process_common (ioam_export_main_t * em, vlib_main_t * vm,
 345                             vlib_node_runtime_t * rt, vlib_frame_t * f,
 346                             u32 index)
 347 {
 348   f64 now;
 349   f64 timeout = 30.0;
 350   uword event_type;
 351   uword *event_data = 0;
 352   int i;
 353   ioam_export_buffer_t *eb = 0, *new_eb = 0;
 354   u32 *vec_buffer_indices = 0;
 355   u32 *vec_buffer_to_be_sent = 0;
 356   u32 *thread_index = 0;
 357   u32 new_pool_index = 0;
 358
 359   em->export_process_node_index = index;
 360   /* Wait for Godot... */
 361   vlib_process_wait_for_event_or_clock (vm, 1e9);
 362   event_type = vlib_process_get_events (vm, &event_data);
 363   if (event_type != 1)
 364     clib_warning ("bogus kickoff event received, %d", event_type);
 365   vec_reset_length (event_data);
 366
 367   while (1)
 368     {
 369       vlib_process_wait_for_event_or_clock (vm, timeout);
 370       event_type = vlib_process_get_events (vm, &event_data);
 371       switch (event_type)
 372         {
 373         case 2:         /* Stop and Wait for kickoff again */
 374           timeout = 1e9;
 375           break;
 376         case 1:         /* kickoff : Check for unsent buffers */
 377           timeout = THREAD_PERIOD;
 378           break;
 379         case ~0:                /* timeout */
 380           break;
 381         }
 382       vec_reset_length (event_data);
 383       now = vlib_time_now (vm);
 384       /*
 385        * Create buffers for threads that are not active enough
 386        * to send out the export records
 387        */
 388       for (i = 0; i < vec_len (em->buffer_per_thread); i++)
 389         {
 390           /* If the worker thread is processing export records ignore further checks */
 391           if (*em->lockp[i] == 1)
 392             continue;
 393           eb = pool_elt_at_index (em->buffer_pool, em->buffer_per_thread[i]);
 394           if (eb->records_in_this_buffer > 0
 395               && now > (eb->touched_at + EXPORT_TIMEOUT))
 396             {
 397               pool_get_aligned (em->buffer_pool, new_eb,
 398                                 CLIB_CACHE_LINE_BYTES);
 399               memset (new_eb, 0, sizeof (*new_eb));
 400               if (ioam_export_init_buffer (em, vm, new_eb) == 1)
 401                 {
 402                   new_pool_index = new_eb - em->buffer_pool;
 403                   vec_add (vec_buffer_indices, &new_pool_index, 1);
 404                   vec_add (vec_buffer_to_be_sent, &em->buffer_per_thread[i],
 405                            1);
 406                   vec_add (thread_index, &i, 1);
 407                 }
 408               else
 409                 {
 410                   pool_put (em->buffer_pool, new_eb);
 411                   /*Give up */
 412                   goto CLEANUP;
 413                 }
 414             }
 415         }
 416       if (vec_len (thread_index) != 0)
 417         {
 418           /*
 419            * Now swap the buffers out
 420            */
 421           for (i = 0; i < vec_len (thread_index); i++)
 422             {
 423               while (__sync_lock_test_and_set (em->lockp[thread_index[i]], 1))
 424                 ;
 425               em->buffer_per_thread[thread_index[i]] =
 426                 vec_pop (vec_buffer_indices);
 427               *em->lockp[thread_index[i]] = 0;
 428             }
 429
 430           /* Send the buffers */
 431           for (i = 0; i < vec_len (vec_buffer_to_be_sent); i++)
 432             {
 433               eb =
 434                 pool_elt_at_index (em->buffer_pool, vec_buffer_to_be_sent[i]);
 435               ioam_export_send_buffer (em, vm, eb);
 436               pool_put (em->buffer_pool, eb);
 437             }
 438         }
 439
 440     CLEANUP:
 441       /* Free any leftover/unused buffers and everything that was allocated */
 442       for (i = 0; i < vec_len (vec_buffer_indices); i++)
 443         {
 444           new_eb = pool_elt_at_index (em->buffer_pool, vec_buffer_indices[i]);
 445           vlib_buffer_free (vm, &new_eb->buffer_index, 1);
 446           pool_put (em->buffer_pool, new_eb);
 447         }
 448       vec_free (vec_buffer_indices);
 449       vec_free (vec_buffer_to_be_sent);
 450       vec_free (thread_index);
 451     }
 452   return 0;                     /* not so much */
 453 }
 454
 455 #define ioam_export_node_common(EM, VM, N, F, HTYPE, L, V, NEXT)               \
 456 do {                                                                           \
 457   u32 n_left_from, *from, *to_next;                                            \
 458   export_next_t next_index;                                                    \
 459   u32 pkts_recorded = 0;                                                       \
 460   ioam_export_buffer_t *my_buf = 0;                                            \
 461   vlib_buffer_t *eb0 = 0;                                                      \
 462   u32 ebi0 = 0;                                                                \
 463   from = vlib_frame_vector_args (F);                                           \
 464   n_left_from = (F)->n_vectors;                                                \
 465   next_index = (N)->cached_next_index;                                         \
 466   while (__sync_lock_test_and_set ((EM)->lockp[(VM)->cpu_index], 1));          \
 467   my_buf = ioam_export_get_my_buffer (EM, (VM)->cpu_index);                    \
 468   my_buf->touched_at = vlib_time_now (VM);                                     \
 469   while (n_left_from > 0)                                                      \
 470     {                                                                          \
 471       u32 n_left_to_next;                                                      \
 472       vlib_get_next_frame (VM, N, next_index, to_next, n_left_to_next);        \
 473       while (n_left_from >= 4 && n_left_to_next >= 2)                          \
 474         {                                                                      \
 475           u32 next0 = NEXT;                                                    \
 476           u32 next1 = NEXT;                                                    \
 477           u32 bi0, bi1;                                                        \
 478           HTYPE *ip0, *ip1;                                                    \
 479           vlib_buffer_t *p0, *p1;                                              \
 480           u32 ip_len0, ip_len1;                                                \
 481           {                                                                    \
 482             vlib_buffer_t *p2, *p3;                                            \
 483             p2 = vlib_get_buffer (VM, from[2]);                                \
 484             p3 = vlib_get_buffer (VM, from[3]);                                \
 485             vlib_prefetch_buffer_header (p2, LOAD);                            \
 486             vlib_prefetch_buffer_header (p3, LOAD);                            \
 487             CLIB_PREFETCH (p2->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD);         \
 488             CLIB_PREFETCH (p3->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD);         \
 489           }                                                                    \
 490           to_next[0] = bi0 = from[0];                                          \
 491           to_next[1] = bi1 = from[1];                                          \
 492           from += 2;                                                           \
 493           to_next += 2;                                                        \
 494           n_left_from -= 2;                                                    \
 495           n_left_to_next -= 2;                                                 \
 496           p0 = vlib_get_buffer (VM, bi0);                                      \
 497           p1 = vlib_get_buffer (VM, bi1);                                      \
 498           ip0 = vlib_buffer_get_current (p0);                                  \
 499           ip1 = vlib_buffer_get_current (p1);                                  \
 500           ip_len0 =                                                            \
 501             clib_net_to_host_u16 (ip0->L) + sizeof (HTYPE);                    \
 502           ip_len1 =                                                            \
 503             clib_net_to_host_u16 (ip1->L) + sizeof (HTYPE);                    \
 504           ebi0 = my_buf->buffer_index;                                         \
 505           eb0 = vlib_get_buffer (VM, ebi0);                                    \
 506           if (PREDICT_FALSE (eb0 == 0))                                        \
 507             goto NO_BUFFER1;                                                   \
 508           ip_len0 =                                                            \
 509             ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0;     \
 510           ip_len1 =                                                            \
 511             ip_len1 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len1;     \
 512           copy3cachelines (eb0->data + eb0->current_length, ip0, ip_len0);     \
 513           eb0->current_length += DEFAULT_EXPORT_SIZE;                          \
 514           my_buf->records_in_this_buffer++;                                    \
 515           if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS)        \
 516             {                                                                  \
 517               ioam_export_send_buffer (EM, VM, my_buf);                        \
 518               ioam_export_init_buffer (EM, VM, my_buf);                        \
 519             }                                                                  \
 520           ebi0 = my_buf->buffer_index;                                         \
 521           eb0 = vlib_get_buffer (VM, ebi0);                                    \
 522           if (PREDICT_FALSE (eb0 == 0))                                        \
 523             goto NO_BUFFER1;                                                   \
 524           copy3cachelines (eb0->data + eb0->current_length, ip1, ip_len1);     \
 525           eb0->current_length += DEFAULT_EXPORT_SIZE;                          \
 526           my_buf->records_in_this_buffer++;                                    \
 527           if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS)        \
 528             {                                                                  \
 529               ioam_export_send_buffer (EM, VM, my_buf);                        \
 530               ioam_export_init_buffer (EM, VM, my_buf);                        \
 531             }                                                                  \
 532           pkts_recorded += 2;                                                  \
 533           if (PREDICT_FALSE (((node)->flags & VLIB_NODE_FLAG_TRACE)))          \
 534             {                                                                  \
 535               if (p0->flags & VLIB_BUFFER_IS_TRACED)                           \
 536                 {                                                              \
 537                   export_trace_t *t =                                          \
 538                     vlib_add_trace (VM, node, p0, sizeof (*t));                \
 539                   t->flow_label =                                              \
 540                     clib_net_to_host_u32 (ip0->V);                             \
 541                   t->next_index = next0;                                       \
 542                 }                                                              \
 543               if (p1->flags & VLIB_BUFFER_IS_TRACED)                           \
 544                 {                                                              \
 545                   export_trace_t *t =                                          \
 546                     vlib_add_trace (VM, N, p1, sizeof (*t));                   \
 547                   t->flow_label =                                              \
 548                     clib_net_to_host_u32 (ip1->V);                             \
 549                   t->next_index = next1;                                       \
 550                 }                                                              \
 551             }                                                                  \
 552         NO_BUFFER1:                                                            \
 553           vlib_validate_buffer_enqueue_x2 (VM, N, next_index,                  \
 554                                            to_next, n_left_to_next,            \
 555                                            bi0, bi1, next0, next1);            \
 556         }                                                                      \
 557       while (n_left_from > 0 && n_left_to_next > 0)                            \
 558         {                                                                      \
 559           u32 bi0;                                                             \
 560           vlib_buffer_t *p0;                                                   \
 561           u32 next0 = NEXT;                                                    \
 562           HTYPE *ip0;                                                          \
 563           u32 ip_len0;                                                         \
 564           bi0 = from[0];                                                       \
 565           to_next[0] = bi0;                                                    \
 566           from += 1;                                                           \
 567           to_next += 1;                                                        \
 568           n_left_from -= 1;                                                    \
 569           n_left_to_next -= 1;                                                 \
 570           p0 = vlib_get_buffer (VM, bi0);                                      \
 571           ip0 = vlib_buffer_get_current (p0);                                  \
 572           ip_len0 =                                                            \
 573             clib_net_to_host_u16 (ip0->L) + sizeof (HTYPE);                    \
 574           ebi0 = my_buf->buffer_index;                                         \
 575           eb0 = vlib_get_buffer (VM, ebi0);                                    \
 576           if (PREDICT_FALSE (eb0 == 0))                                        \
 577             goto NO_BUFFER;                                                    \
 578           ip_len0 =                                                            \
 579             ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0;     \
 580           copy3cachelines (eb0->data + eb0->current_length, ip0, ip_len0);     \
 581           eb0->current_length += DEFAULT_EXPORT_SIZE;                          \
 582           my_buf->records_in_this_buffer++;                                    \
 583           if (my_buf->records_in_this_buffer >= DEFAULT_EXPORT_RECORDS)        \
 584             {                                                                  \
 585               ioam_export_send_buffer (EM, VM, my_buf);                        \
 586               ioam_export_init_buffer (EM, VM, my_buf);                        \
 587             }                                                                  \
 588           if (PREDICT_FALSE (((N)->flags & VLIB_NODE_FLAG_TRACE)               \
 589                              && (p0->flags & VLIB_BUFFER_IS_TRACED)))          \
 590             {                                                                  \
 591               export_trace_t *t = vlib_add_trace (VM, (N), p0, sizeof (*t));   \
 592               t->flow_label =                                                  \
 593                 clib_net_to_host_u32 (ip0->V);                                 \
 594               t->next_index = next0;                                           \
 595             }                                                                  \
 596           pkts_recorded += 1;                                                  \
 597         NO_BUFFER:                                                             \
 598           vlib_validate_buffer_enqueue_x1 (VM, N, next_index,                  \
 599                                            to_next, n_left_to_next,            \
 600                                            bi0, next0);                        \
 601         }                                                                      \
 602       vlib_put_next_frame (VM, N, next_index, n_left_to_next);                 \
 603     }                                                                          \
 604   vlib_node_increment_counter (VM, export_node.index,                          \
 605                                EXPORT_ERROR_RECORDED, pkts_recorded);          \
 606   *(EM)->lockp[(VM)->cpu_index] = 0;                                           \
 607 } while(0)
 608
 609 #endif /* __included_ioam_export_h__ */
 610
 611 /*
 612  * fd.io coding-style-patch-verification: ON
 613  *
 614  * Local Variables:
 615  * eval: (c-set-style "gnu")
 616  * End:
 617  */