X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fmemif%2Fmemif.c;h=22f3e3dcdf162b202be16404d63877c023bc6964;hb=34c721fb47155135bf2173ca7b9a31aaacfde190;hp=12d81ee79754034b1319c4d8a2964782004b11b5;hpb=ab4d9174d890bff4c07b44957a20eacb33c88172;p=vpp.git

diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c
index 12d81ee7975..22f3e3dcdf1 100644
--- a/src/plugins/memif/memif.c
+++ b/src/plugins/memif/memif.c
@@ -100,6 +100,8 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err)
   memif_region_t *mr;
   memif_queue_t *mq;
   int i;
+  vlib_main_t *vm = vlib_get_main ();
+  int with_barrier = 0;
 
   if (mif == 0)
     return;
@@ -141,6 +143,12 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err)
       clib_mem_free (mif->sock);
     }
 
+  if (vlib_worker_thread_barrier_held () == 0)
+    {
+      with_barrier = 1;
+      vlib_worker_thread_barrier_sync (vm);
+    }
+
   /* *INDENT-OFF* */
   vec_foreach_index (i, mif->rx_queues)
     {
@@ -198,6 +206,9 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err)
   vec_free (mif->remote_name);
   vec_free (mif->remote_if_name);
   clib_fifo_free (mif->msg_queue);
+
+  if (with_barrier)
+    vlib_worker_thread_barrier_release (vm);
 }
 
 static clib_error_t *
@@ -301,6 +312,37 @@ memif_connect (memif_if_t * mif)
       mq->queue_index =
 	vnet_hw_if_register_tx_queue (vnm, mif->hw_if_index, i);
       clib_spinlock_init (&mq->lockp);
+
+      if (mif->flags & MEMIF_IF_FLAG_USE_DMA)
+	{
+	  memif_dma_info_t *dma_info;
+	  mq->dma_head = 0;
+	  mq->dma_tail = 0;
+	  mq->dma_info_head = 0;
+	  mq->dma_info_tail = 0;
+	  mq->dma_info_size = MEMIF_DMA_INFO_SIZE;
+	  vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE,
+				CLIB_CACHE_LINE_BYTES);
+
+	  vec_foreach (dma_info, mq->dma_info)
+	    {
+	      vec_validate_aligned (dma_info->data.desc_data,
+				    pow2_mask (max_log2_ring_sz),
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_validate_aligned (dma_info->data.desc_len,
+				    pow2_mask (max_log2_ring_sz),
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_validate_aligned (dma_info->data.desc_status,
+				    pow2_mask (max_log2_ring_sz),
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_validate_aligned (dma_info->data.copy_ops, 0,
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_reset_length (dma_info->data.copy_ops);
+	      vec_validate_aligned (dma_info->data.buffers, 0,
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_reset_length (dma_info->data.buffers);
+	    }
+	}
     }
 
   if (vec_len (mif->tx_queues) > 0)
@@ -331,6 +373,37 @@ memif_connect (memif_if_t * mif)
       qi = vnet_hw_if_register_rx_queue (vnm, mif->hw_if_index, i,
 					 VNET_HW_IF_RXQ_THREAD_ANY);
       mq->queue_index = qi;
+
+      if (mif->flags & MEMIF_IF_FLAG_USE_DMA)
+	{
+	  memif_dma_info_t *dma_info;
+	  mq->dma_head = 0;
+	  mq->dma_tail = 0;
+	  mq->dma_info_head = 0;
+	  mq->dma_info_tail = 0;
+	  mq->dma_info_size = MEMIF_DMA_INFO_SIZE;
+	  vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE,
+				CLIB_CACHE_LINE_BYTES);
+	  vec_foreach (dma_info, mq->dma_info)
+	    {
+	      vec_validate_aligned (dma_info->data.desc_data,
+				    pow2_mask (max_log2_ring_sz),
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_validate_aligned (dma_info->data.desc_len,
+				    pow2_mask (max_log2_ring_sz),
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_validate_aligned (dma_info->data.desc_status,
+				    pow2_mask (max_log2_ring_sz),
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_validate_aligned (dma_info->data.copy_ops, 0,
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_reset_length (dma_info->data.copy_ops);
+	      vec_validate_aligned (dma_info->data.buffers, 0,
+				    CLIB_CACHE_LINE_BYTES);
+	      vec_reset_length (dma_info->data.buffers);
+	    }
+	}
+
       if (mq->int_fd > -1)
 	{
 	  template.file_descriptor = mq->int_fd;
@@ -641,8 +714,8 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
 	    {
               clib_memset (sock, 0, sizeof(clib_socket_t));
 	      sock->config = (char *) msf->filename;
-	      sock->flags = CLIB_SOCKET_F_IS_CLIENT | CLIB_SOCKET_F_SEQPACKET |
-			    CLIB_SOCKET_F_BLOCKING;
+	      sock->is_seqpacket = 1;
+	      sock->is_blocking = 1;
 
 	      if ((err = clib_socket_init (sock)))
 		{
@@ -675,79 +748,52 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (memif_process_node,static) = {
   .function = memif_process,
   .type = VLIB_NODE_TYPE_PROCESS,
   .name = "memif-process",
 };
-/* *INDENT-ON* */
 
-static clib_error_t *
-memif_add_socket_file (u32 sock_id, u8 *socket_filename)
+/*
+ * Returns an unused socket id, and ~0 if it can't find one.
+ */
+u32
+memif_get_unused_socket_id ()
 {
   memif_main_t *mm = &memif_main;
   uword *p;
-  memif_socket_file_t *msf;
+  int i, j;
 
-  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
-  if (p)
+  static u32 seed = 0;
+  /* limit to 1M tries */
+  for (j = 0; j < 1 << 10; j++)
     {
-      msf = pool_elt_at_index (mm->socket_files, *p);
-      if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0)
+      seed = random_u32 (&seed);
+      for (i = 0; i < 1 << 10; i++)
 	{
-	  /* Silently accept identical "add". */
-	  return 0;
+	  /* look around randomly generated id */
+	  seed += (2 * (i % 2) - 1) * i;
+	  if (seed == (u32) ~0)
+	    continue;
+	  p = hash_get (mm->socket_file_index_by_sock_id, seed);
+	  if (!p)
+	    return seed;
 	}
-
-      /* But don't allow a direct add of a different filename. */
-      return vnet_error (VNET_ERR_ENTRY_ALREADY_EXISTS,
-			 "entry already exists");
     }
 
-  pool_get (mm->socket_files, msf);
-  clib_memset (msf, 0, sizeof (memif_socket_file_t));
-
-  msf->filename = socket_filename;
-  msf->socket_id = sock_id;
-
-  hash_set (mm->socket_file_index_by_sock_id, sock_id,
-	    msf - mm->socket_files);
-
-  return 0;
+  return ~0;
 }
 
-static clib_error_t *
-memif_delete_socket_file (u32 sock_id)
+clib_error_t *
+memif_socket_filename_add_del (u8 is_add, u32 sock_id, char *sock_filename)
 {
   memif_main_t *mm = &memif_main;
   uword *p;
   memif_socket_file_t *msf;
-
-  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
-  if (!p)
-    /* Don't delete non-existent entries. */
-    return vnet_error (VNET_ERR_INVALID_ARGUMENT,
-		       "socket file with id %u does not exist", sock_id);
-
-  msf = pool_elt_at_index (mm->socket_files, *p);
-  if (msf->ref_cnt > 0)
-    return vnet_error (VNET_ERR_UNEXPECTED_INTF_STATE,
-		       "socket file '%s' is in use", msf->filename);
-
-  vec_free (msf->filename);
-  pool_put (mm->socket_files, msf);
-
-  hash_unset (mm->socket_file_index_by_sock_id, sock_id);
-
-  return 0;
-}
-
-clib_error_t *
-memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 *sock_filename)
-{
+  clib_error_t *err = 0;
   char *dir = 0, *tmp;
   u32 idx = 0;
+  u8 *name = 0;
 
   /* allow adding socket id 0 */
   if (sock_id == 0 && is_add == 0)
@@ -758,70 +804,95 @@ memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 *sock_filename)
 		       "socked id is not specified");
 
   if (is_add == 0)
-    return memif_delete_socket_file (sock_id);
+    {
+      p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
+      if (!p)
+	/* Don't delete non-existent entries. */
+	return vnet_error (VNET_ERR_INVALID_ARGUMENT,
+			   "socket file with id %u does not exist", sock_id);
+
+      msf = pool_elt_at_index (mm->socket_files, *p);
+      if (msf->ref_cnt > 0)
+	return vnet_error (VNET_ERR_UNEXPECTED_INTF_STATE,
+			   "socket file '%s' is in use", msf->filename);
+
+      vec_free (msf->filename);
+      pool_put (mm->socket_files, msf);
+
+      hash_unset (mm->socket_file_index_by_sock_id, sock_id);
+
+      return 0;
+    }
 
   if (sock_filename == 0 || sock_filename[0] == 0)
     return vnet_error (VNET_ERR_INVALID_ARGUMENT,
 		       "socket filename not specified");
 
-  if (sock_filename[0] != '/')
+  if (clib_socket_prefix_is_valid (sock_filename))
+    {
+      name = format (0, "%s%c", sock_filename, 0);
+    }
+  else if (sock_filename[0] == '/')
+    {
+      name = format (0, "%s%c", sock_filename, 0);
+    }
+  else
     {
-      clib_error_t *error;
-
       /* copy runtime dir path */
       vec_add (dir, vlib_unix_get_runtime_dir (),
 	       strlen (vlib_unix_get_runtime_dir ()));
       vec_add1 (dir, '/');
 
       /* if sock_filename contains dirs, add them to path */
-      tmp = strrchr ((char *) sock_filename, '/');
+      tmp = strrchr (sock_filename, '/');
       if (tmp)
 	{
-	  idx = tmp - (char *) sock_filename;
+	  idx = tmp - sock_filename;
 	  vec_add (dir, sock_filename, idx);
 	}
 
       vec_add1 (dir, '\0');
       /* create socket dir */
-      error = vlib_unix_recursive_mkdir (dir);
-      if (error)
+      if ((err = vlib_unix_recursive_mkdir (dir)))
 	{
-	  clib_error_free (error);
-	  return vnet_error (VNET_ERR_SYSCALL_ERROR_1,
-			     "unable to create socket dir");
+	  clib_error_free (err);
+	  err = vnet_error (VNET_ERR_SYSCALL_ERROR_1,
+			    "unable to create socket dir");
+	  goto done;
 	}
 
-      sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (),
-			      sock_filename, 0);
+      name =
+	format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), sock_filename, 0);
     }
-  else
-    {
-      sock_filename = vec_dup (sock_filename);
 
-      /* check if directory exists */
-      tmp = strrchr ((char *) sock_filename, '/');
-      if (tmp)
+  p = hash_get (mm->socket_file_index_by_sock_id, sock_id);
+  if (p)
+    {
+      msf = pool_elt_at_index (mm->socket_files, *p);
+      if (strcmp ((char *) msf->filename, (char *) name) == 0)
 	{
-	  idx = tmp - (char *) sock_filename;
-	  vec_add (dir, sock_filename, idx);
-	  vec_add1 (dir, '\0');
+	  /* Silently accept identical "add". */
+	  goto done;
 	}
 
-      /* check dir existance and access rights for effective user/group IDs */
-      if ((dir == NULL)
-	  ||
-	  (faccessat ( /* ignored */ -1, dir, F_OK | R_OK | W_OK, AT_EACCESS)
-	   < 0))
-	{
-	  vec_free (dir);
-	  return vnet_error (
-	    VNET_ERR_INVALID_ARGUMENT,
-	    "directory doesn't exist or no access permissions");
-	}
+      /* But don't allow a direct add of a different filename. */
+      err = vnet_error (VNET_ERR_ENTRY_ALREADY_EXISTS, "entry already exists");
+      goto done;
     }
-  vec_free (dir);
 
-  return memif_add_socket_file (sock_id, sock_filename);
+  pool_get (mm->socket_files, msf);
+  clib_memset (msf, 0, sizeof (memif_socket_file_t));
+
+  msf->filename = name;
+  msf->socket_id = sock_id;
+  name = 0;
+
+  hash_set (mm->socket_file_index_by_sock_id, sock_id, msf - mm->socket_files);
+
+done:
+  vec_free (name);
+  vec_free (dir);
+  return err;
 }
 
 clib_error_t *
@@ -904,6 +975,16 @@ VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = {
 };
 /* *INDENT-ON* */
 
+static void
+memif_prepare_dma_args (vlib_dma_config_t *args)
+{
+  args->max_batches = 256;
+  args->max_transfer_size = VLIB_BUFFER_DEFAULT_DATA_SIZE;
+  args->barrier_before_last = 1;
+  args->sw_fallback = 1;
+  args->callback_fn = NULL;
+}
+
 clib_error_t *
 memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args)
 {
@@ -949,24 +1030,6 @@ memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args)
   /* Create new socket file */
   if (msf->ref_cnt == 0)
     {
-      struct stat file_stat;
-
-      /* If we are creating listener make sure file doesn't exist or if it
-       * exists thn delete it if it is old socket file */
-      if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0))
-	{
-	  if (S_ISSOCK (file_stat.st_mode))
-	    {
-	      unlink ((char *) msf->filename);
-	    }
-	  else
-	    {
-	      err = vnet_error (VNET_ERR_VALUE_EXIST, "File exists for %s",
-				msf->filename);
-	      goto done;
-	    }
-	}
-
       mhash_init (&msf->dev_instance_by_id, sizeof (uword),
 		  sizeof (memif_interface_id_t));
       msf->dev_instance_by_fd = hash_create (0, sizeof (uword));
@@ -1009,6 +1072,20 @@ memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args)
   if (args->secret)
     mif->secret = vec_dup (args->secret);
 
+  /* register dma config if enabled */
+  if (args->use_dma)
+    {
+      vlib_dma_config_t dma_args;
+      bzero (&dma_args, sizeof (dma_args));
+      memif_prepare_dma_args (&dma_args);
+
+      dma_args.max_transfers = 1 << args->log2_ring_size;
+      dma_args.callback_fn = memif_dma_completion_cb;
+      mif->dma_input_config = vlib_dma_config_add (vm, &dma_args);
+      dma_args.callback_fn = memif_tx_dma_completion_cb;
+      mif->dma_tx_config = vlib_dma_config_add (vm, &dma_args);
+    }
+
   if (mif->mode == MEMIF_INTERFACE_MODE_ETHERNET)
     {
 
@@ -1068,9 +1145,11 @@ memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args)
 
       clib_memset (s, 0, sizeof (clib_socket_t));
       s->config = (char *) msf->filename;
-      s->flags = CLIB_SOCKET_F_IS_SERVER |
-	CLIB_SOCKET_F_ALLOW_GROUP_WRITE |
-	CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED;
+      s->local_only = 1;
+      s->is_server = 1;
+      s->allow_group_write = 1;
+      s->is_seqpacket = 1;
+      s->passcred = 1;
 
       if ((err = clib_socket_init (s)))
 	{
@@ -1095,6 +1174,9 @@ memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args)
 	mif->flags |= MEMIF_IF_FLAG_ZERO_COPY;
     }
 
+  if (args->use_dma)
+    mif->flags |= MEMIF_IF_FLAG_USE_DMA;
+
   vnet_hw_if_set_caps (vnm, mif->hw_if_index, VNET_HW_IF_CAP_INT_MODE);
   vnet_hw_if_set_input_node (vnm, mif->hw_if_index, memif_input_node.index);
   mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0);
@@ -1159,9 +1241,7 @@ memif_init (vlib_main_t * vm)
    * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the
    * default run-time directory.
    */
-  memif_socket_filename_add_del (1, 0, (u8 *) MEMIF_DEFAULT_SOCKET_FILENAME);
-
-  return 0;
+  return memif_socket_filename_add_del (1, 0, MEMIF_DEFAULT_SOCKET_FILENAME);
 }
 
 VLIB_INIT_FUNCTION (memif_init);