vnet/vnet/handoff.c

   1
   2 /*
   3  * Copyright (c) 2016 Cisco and/or its affiliates.
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <vnet/vnet.h>
  18 #include <vppinfra/xxhash.h>
  19 #include <vlib/threads.h>
  20 #include <vnet/handoff.h>
  21 #include <vnet/feature/feature.h>
  22
  23 typedef struct
  24 {
  25   uword *workers_bitmap;
  26   u32 *workers;
  27 } per_inteface_handoff_data_t;
  28
  29 typedef struct
  30 {
  31   u32 cached_next_index;
  32   u32 num_workers;
  33   u32 first_worker_index;
  34
  35   per_inteface_handoff_data_t *if_data;
  36
  37   /* convenience variables */
  38   vlib_main_t *vlib_main;
  39   vnet_main_t *vnet_main;
  40 } handoff_main_t;
  41
  42 handoff_main_t handoff_main;
  43
  44 typedef struct
  45 {
  46   u32 sw_if_index;
  47   u32 next_worker_index;
  48   u32 buffer_index;
  49 } worker_handoff_trace_t;
  50
  51 /* packet trace format function */
  52 static u8 *
  53 format_worker_handoff_trace (u8 * s, va_list * args)
  54 {
  55   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  56   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  57   worker_handoff_trace_t *t = va_arg (*args, worker_handoff_trace_t *);
  58
  59   s =
  60     format (s, "worker-handoff: sw_if_index %d, next_worker %d, buffer 0x%x",
  61             t->sw_if_index, t->next_worker_index, t->buffer_index);
  62   return s;
  63 }
  64
  65 vlib_node_registration_t handoff_node;
  66
  67 static uword
  68 worker_handoff_node_fn (vlib_main_t * vm,
  69                         vlib_node_runtime_t * node, vlib_frame_t * frame)
  70 {
  71   handoff_main_t *hm = &handoff_main;
  72   vlib_thread_main_t *tm = vlib_get_thread_main ();
  73   u32 n_left_from, *from;
  74   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
  75   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
  76     = 0;
  77   vlib_frame_queue_elt_t *hf = 0;
  78   int i;
  79   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
  80   u32 next_worker_index = 0;
  81   u32 current_worker_index = ~0;
  82
  83   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
  84     {
  85       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
  86
  87       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
  88                                hm->first_worker_index + hm->num_workers - 1,
  89                                (vlib_frame_queue_t *) (~0));
  90     }
  91
  92   from = vlib_frame_vector_args (frame);
  93   n_left_from = frame->n_vectors;
  94
  95   while (n_left_from > 0)
  96     {
  97       u32 bi0;
  98       vlib_buffer_t *b0;
  99       u32 sw_if_index0;
 100       u32 hash;
 101       u64 hash_key;
 102       per_inteface_handoff_data_t *ihd0;
 103       u32 index0;
 104
 105       bi0 = from[0];
 106       from += 1;
 107       n_left_from -= 1;
 108
 109       b0 = vlib_get_buffer (vm, bi0);
 110       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 111       ASSERT (hm->if_data);
 112       ihd0 = vec_elt_at_index (hm->if_data, sw_if_index0);
 113
 114       next_worker_index = hm->first_worker_index;
 115
 116       /*
 117        * Force unknown traffic onto worker 0,
 118        * and into ethernet-input. $$$$ add more hashes.
 119        */
 120
 121       /* Compute ingress LB hash */
 122       hash_key = eth_get_key ((ethernet_header_t *) b0->data);
 123       hash = (u32) clib_xxhash (hash_key);
 124
 125       /* if input node did not specify next index, then packet
 126          should go to eternet-input */
 127       if (PREDICT_FALSE ((b0->flags & BUFFER_HANDOFF_NEXT_VALID) == 0))
 128         vnet_buffer (b0)->handoff.next_index =
 129           HANDOFF_DISPATCH_NEXT_ETHERNET_INPUT;
 130       else if (vnet_buffer (b0)->handoff.next_index ==
 131                HANDOFF_DISPATCH_NEXT_IP4_INPUT
 132                || vnet_buffer (b0)->handoff.next_index ==
 133                HANDOFF_DISPATCH_NEXT_IP6_INPUT
 134                || vnet_buffer (b0)->handoff.next_index ==
 135                HANDOFF_DISPATCH_NEXT_MPLS_INPUT)
 136         vlib_buffer_advance (b0, (sizeof (ethernet_header_t)));
 137
 138       if (PREDICT_TRUE (is_pow2 (vec_len (ihd0->workers))))
 139         index0 = hash & (vec_len (ihd0->workers) - 1);
 140       else
 141         index0 = hash % vec_len (ihd0->workers);
 142
 143       next_worker_index += ihd0->workers[index0];
 144
 145       if (next_worker_index != current_worker_index)
 146         {
 147           if (hf)
 148             hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 149
 150           hf = dpdk_get_handoff_queue_elt (next_worker_index,
 151                                            handoff_queue_elt_by_worker_index);
 152
 153           n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
 154           to_next_worker = &hf->buffer_index[hf->n_vectors];
 155           current_worker_index = next_worker_index;
 156         }
 157
 158       /* enqueue to correct worker thread */
 159       to_next_worker[0] = bi0;
 160       to_next_worker++;
 161       n_left_to_next_worker--;
 162
 163       if (n_left_to_next_worker == 0)
 164         {
 165           hf->n_vectors = VLIB_FRAME_SIZE;
 166           vlib_put_handoff_queue_elt (hf);
 167           current_worker_index = ~0;
 168           handoff_queue_elt_by_worker_index[next_worker_index] = 0;
 169           hf = 0;
 170         }
 171
 172       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 173                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 174         {
 175           worker_handoff_trace_t *t =
 176             vlib_add_trace (vm, node, b0, sizeof (*t));
 177           t->sw_if_index = sw_if_index0;
 178           t->next_worker_index = next_worker_index - hm->first_worker_index;
 179           t->buffer_index = bi0;
 180         }
 181
 182     }
 183
 184   if (hf)
 185     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
 186
 187   /* Ship frames to the worker nodes */
 188   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
 189     {
 190       if (handoff_queue_elt_by_worker_index[i])
 191         {
 192           hf = handoff_queue_elt_by_worker_index[i];
 193           /*
 194            * It works better to let the handoff node
 195            * rate-adapt, always ship the handoff queue element.
 196            */
 197           if (1 || hf->n_vectors == hf->last_n_vectors)
 198             {
 199               vlib_put_handoff_queue_elt (hf);
 200               handoff_queue_elt_by_worker_index[i] = 0;
 201             }
 202           else
 203             hf->last_n_vectors = hf->n_vectors;
 204         }
 205       congested_handoff_queue_by_worker_index[i] =
 206         (vlib_frame_queue_t *) (~0);
 207     }
 208   hf = 0;
 209   current_worker_index = ~0;
 210   return frame->n_vectors;
 211 }
 212
 213 /* *INDENT-OFF* */
 214 VLIB_REGISTER_NODE (worker_handoff_node) = {
 215   .function = worker_handoff_node_fn,
 216   .name = "worker-handoff",
 217   .vector_size = sizeof (u32),
 218   .format_trace = format_worker_handoff_trace,
 219   .type = VLIB_NODE_TYPE_INTERNAL,
 220
 221   .n_next_nodes = 1,
 222   .next_nodes = {
 223     [0] = "error-drop",
 224   },
 225 };
 226
 227 VLIB_NODE_FUNCTION_MULTIARCH (worker_handoff_node, worker_handoff_node_fn)
 228 /* *INDENT-ON* */
 229
 230 int
 231 interface_handoff_enable_disable (vlib_main_t * vm, u32 sw_if_index,
 232                                   uword * bitmap, int enable_disable)
 233 {
 234   handoff_main_t *hm = &handoff_main;
 235   vnet_sw_interface_t *sw;
 236   vnet_main_t *vnm = vnet_get_main ();
 237   per_inteface_handoff_data_t *d;
 238   int i, rv = 0;
 239
 240   if (pool_is_free_index (vnm->interface_main.sw_interfaces, sw_if_index))
 241     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 242
 243   sw = vnet_get_sw_interface (vnm, sw_if_index);
 244   if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
 245     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 246
 247   if (clib_bitmap_last_set (bitmap) >= hm->num_workers)
 248     return VNET_API_ERROR_INVALID_WORKER;
 249
 250   vec_validate (hm->if_data, sw_if_index);
 251   d = vec_elt_at_index (hm->if_data, sw_if_index);
 252
 253   vec_free (d->workers);
 254   vec_free (d->workers_bitmap);
 255
 256   if (enable_disable)
 257     {
 258       d->workers_bitmap = bitmap;
 259       /* *INDENT-OFF* */
 260       clib_bitmap_foreach (i, bitmap,
 261         ({
 262           vec_add1(d->workers, i);
 263         }));
 264       /* *INDENT-ON* */
 265     }
 266
 267   vnet_feature_enable_disable ("device-input", "worker-handoff",
 268                                sw_if_index, enable_disable, 0, 0);
 269   return rv;
 270 }
 271
 272 static clib_error_t *
 273 set_interface_handoff_command_fn (vlib_main_t * vm,
 274                                   unformat_input_t * input,
 275                                   vlib_cli_command_t * cmd)
 276 {
 277   u32 sw_if_index = ~0;
 278   int enable_disable = 1;
 279   uword *bitmap = 0;
 280
 281   int rv = 0;
 282
 283   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
 284     {
 285       if (unformat (input, "disable"))
 286         enable_disable = 0;
 287       else if (unformat (input, "workers %U", unformat_bitmap_list, &bitmap))
 288         ;
 289       else if (unformat (input, "%U", unformat_vnet_sw_interface,
 290                          vnet_get_main (), &sw_if_index))
 291         ;
 292       else
 293         break;
 294     }
 295
 296   if (sw_if_index == ~0)
 297     return clib_error_return (0, "Please specify an interface...");
 298
 299   if (bitmap == 0)
 300     return clib_error_return (0, "Please specify list of workers...");
 301
 302   rv =
 303     interface_handoff_enable_disable (vm, sw_if_index, bitmap,
 304                                       enable_disable);
 305
 306   switch (rv)
 307     {
 308     case 0:
 309       break;
 310
 311     case VNET_API_ERROR_INVALID_SW_IF_INDEX:
 312       return clib_error_return (0, "Invalid interface");
 313       break;
 314
 315     case VNET_API_ERROR_INVALID_WORKER:
 316       return clib_error_return (0, "Invalid worker(s)");
 317       break;
 318
 319     case VNET_API_ERROR_UNIMPLEMENTED:
 320       return clib_error_return (0,
 321                                 "Device driver doesn't support redirection");
 322       break;
 323
 324     default:
 325       return clib_error_return (0, "unknown return value %d", rv);
 326     }
 327   return 0;
 328 }
 329
 330 /* *INDENT-OFF* */
 331 VLIB_CLI_COMMAND (set_interface_handoff_command, static) = {
 332   .path = "set interface handoff",
 333   .short_help =
 334   "set interface handoff <interface-name> workers <workers-list>",
 335   .function = set_interface_handoff_command_fn,
 336 };
 337 /* *INDENT-ON* */
 338
 339 typedef struct
 340 {
 341   u32 buffer_index;
 342   u32 next_index;
 343   u32 sw_if_index;
 344 } handoff_dispatch_trace_t;
 345
 346 /* packet trace format function */
 347 static u8 *
 348 format_handoff_dispatch_trace (u8 * s, va_list * args)
 349 {
 350   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 351   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 352   handoff_dispatch_trace_t *t = va_arg (*args, handoff_dispatch_trace_t *);
 353
 354   s = format (s, "handoff-dispatch: sw_if_index %d next_index %d buffer 0x%x",
 355               t->sw_if_index, t->next_index, t->buffer_index);
 356   return s;
 357 }
 358
 359
 360 vlib_node_registration_t handoff_dispatch_node;
 361
 362 #define foreach_handoff_dispatch_error \
 363 _(EXAMPLE, "example packets")
 364
 365 typedef enum
 366 {
 367 #define _(sym,str) HANDOFF_DISPATCH_ERROR_##sym,
 368   foreach_handoff_dispatch_error
 369 #undef _
 370     HANDOFF_DISPATCH_N_ERROR,
 371 } handoff_dispatch_error_t;
 372
 373 static char *handoff_dispatch_error_strings[] = {
 374 #define _(sym,string) string,
 375   foreach_handoff_dispatch_error
 376 #undef _
 377 };
 378
 379 static uword
 380 handoff_dispatch_node_fn (vlib_main_t * vm,
 381                           vlib_node_runtime_t * node, vlib_frame_t * frame)
 382 {
 383   u32 n_left_from, *from, *to_next;
 384   handoff_dispatch_next_t next_index;
 385
 386   from = vlib_frame_vector_args (frame);
 387   n_left_from = frame->n_vectors;
 388   next_index = node->cached_next_index;
 389
 390   while (n_left_from > 0)
 391     {
 392       u32 n_left_to_next;
 393
 394       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 395
 396       while (n_left_from >= 4 && n_left_to_next >= 2)
 397         {
 398           u32 bi0, bi1;
 399           vlib_buffer_t *b0, *b1;
 400           u32 next0, next1;
 401           u32 sw_if_index0, sw_if_index1;
 402
 403           /* Prefetch next iteration. */
 404           {
 405             vlib_buffer_t *p2, *p3;
 406
 407             p2 = vlib_get_buffer (vm, from[2]);
 408             p3 = vlib_get_buffer (vm, from[3]);
 409
 410             vlib_prefetch_buffer_header (p2, LOAD);
 411             vlib_prefetch_buffer_header (p3, LOAD);
 412           }
 413
 414           /* speculatively enqueue b0 and b1 to the current next frame */
 415           to_next[0] = bi0 = from[0];
 416           to_next[1] = bi1 = from[1];
 417           from += 2;
 418           to_next += 2;
 419           n_left_from -= 2;
 420           n_left_to_next -= 2;
 421
 422           b0 = vlib_get_buffer (vm, bi0);
 423           b1 = vlib_get_buffer (vm, bi1);
 424
 425           next0 = vnet_buffer (b0)->handoff.next_index;
 426           next1 = vnet_buffer (b1)->handoff.next_index;
 427
 428           if (PREDICT_FALSE (vm->trace_main.trace_active_hint))
 429             {
 430               if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 431                 {
 432                   vlib_trace_buffer (vm, node, next0, b0,       /* follow_chain */
 433                                      0);
 434                   handoff_dispatch_trace_t *t =
 435                     vlib_add_trace (vm, node, b0, sizeof (*t));
 436                   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 437                   t->sw_if_index = sw_if_index0;
 438                   t->next_index = next0;
 439                   t->buffer_index = bi0;
 440                 }
 441               if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
 442                 {
 443                   vlib_trace_buffer (vm, node, next1, b1,       /* follow_chain */
 444                                      0);
 445                   handoff_dispatch_trace_t *t =
 446                     vlib_add_trace (vm, node, b1, sizeof (*t));
 447                   sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 448                   t->sw_if_index = sw_if_index1;
 449                   t->next_index = next1;
 450                   t->buffer_index = bi1;
 451                 }
 452             }
 453
 454           /* verify speculative enqueues, maybe switch current next frame */
 455           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 456                                            to_next, n_left_to_next,
 457                                            bi0, bi1, next0, next1);
 458         }
 459
 460       while (n_left_from > 0 && n_left_to_next > 0)
 461         {
 462           u32 bi0;
 463           vlib_buffer_t *b0;
 464           u32 next0;
 465           u32 sw_if_index0;
 466
 467           /* speculatively enqueue b0 to the current next frame */
 468           bi0 = from[0];
 469           to_next[0] = bi0;
 470           from += 1;
 471           to_next += 1;
 472           n_left_from -= 1;
 473           n_left_to_next -= 1;
 474
 475           b0 = vlib_get_buffer (vm, bi0);
 476
 477           next0 = vnet_buffer (b0)->handoff.next_index;
 478
 479           if (PREDICT_FALSE (vm->trace_main.trace_active_hint))
 480             {
 481               if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 482                 {
 483                   vlib_trace_buffer (vm, node, next0, b0,       /* follow_chain */
 484                                      0);
 485                   handoff_dispatch_trace_t *t =
 486                     vlib_add_trace (vm, node, b0, sizeof (*t));
 487                   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 488                   t->sw_if_index = sw_if_index0;
 489                   t->next_index = next0;
 490                   t->buffer_index = bi0;
 491                 }
 492             }
 493
 494           /* verify speculative enqueue, maybe switch current next frame */
 495           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 496                                            to_next, n_left_to_next,
 497                                            bi0, next0);
 498         }
 499
 500       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 501     }
 502
 503   return frame->n_vectors;
 504 }
 505
 506 /* *INDENT-OFF* */
 507 VLIB_REGISTER_NODE (handoff_dispatch_node) = {
 508   .function = handoff_dispatch_node_fn,
 509   .name = "handoff-dispatch",
 510   .vector_size = sizeof (u32),
 511   .format_trace = format_handoff_dispatch_trace,
 512   .type = VLIB_NODE_TYPE_INTERNAL,
 513   .flags = VLIB_NODE_FLAG_IS_HANDOFF,
 514
 515   .n_errors = ARRAY_LEN(handoff_dispatch_error_strings),
 516   .error_strings = handoff_dispatch_error_strings,
 517
 518   .n_next_nodes = HANDOFF_DISPATCH_N_NEXT,
 519
 520   .next_nodes = {
 521         [HANDOFF_DISPATCH_NEXT_DROP] = "error-drop",
 522         [HANDOFF_DISPATCH_NEXT_ETHERNET_INPUT] = "ethernet-input",
 523         [HANDOFF_DISPATCH_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
 524         [HANDOFF_DISPATCH_NEXT_IP6_INPUT] = "ip6-input",
 525         [HANDOFF_DISPATCH_NEXT_MPLS_INPUT] = "mpls-input",
 526   },
 527 };
 528
 529 VLIB_NODE_FUNCTION_MULTIARCH (handoff_dispatch_node, handoff_dispatch_node_fn)
 530 /* *INDENT-ON* */
 531
 532 clib_error_t *
 533 handoff_init (vlib_main_t * vm)
 534 {
 535   handoff_main_t *hm = &handoff_main;
 536   vlib_thread_main_t *tm = vlib_get_thread_main ();
 537   clib_error_t *error;
 538   uword *p;
 539
 540   if ((error = vlib_call_init_function (vm, threads_init)))
 541     return error;
 542
 543   vlib_thread_registration_t *tr;
 544   /* Only the standard vnet worker threads are supported */
 545   p = hash_get_mem (tm->thread_registrations_by_name, "workers");
 546   if (p)
 547     {
 548       tr = (vlib_thread_registration_t *) p[0];
 549       if (tr)
 550         {
 551           hm->num_workers = tr->count;
 552           hm->first_worker_index = tr->first_index;
 553         }
 554     }
 555
 556   hm->vlib_main = vm;
 557   hm->vnet_main = &vnet_main;
 558
 559   ASSERT (tm->handoff_dispatch_node_index == ~0);
 560   tm->handoff_dispatch_node_index = handoff_dispatch_node.index;
 561
 562   return 0;
 563 }
 564
 565 VLIB_INIT_FUNCTION (handoff_init);
 566
 567 /*
 568  * fd.io coding-style-patch-verification: ON
 569  *
 570  * Local Variables:
 571  * eval: (c-set-style "gnu")
 572  * End:
 573  */