X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fqueue.c;h=864d97e3de42ebcf235e0dfee11c8ac7b631d5c6;hb=92991e5c690e8ad55f9cf0fdb81c26c1d1143b7d;hp=6972f14cd331f334eee13b2983b0ecd89a33f691;hpb=b7b929931a07fbb27b43d5cd105f366c3e29807e;p=vpp.git

diff --git a/src/svm/queue.c b/src/svm/queue.c
index 6972f14cd33..864d97e3de4 100644
--- a/src/svm/queue.c
+++ b/src/svm/queue.c
@@ -2,7 +2,7 @@
  *------------------------------------------------------------------
  * svm_queue.c - unidirectional shared-memory queues
  *
- * Copyright (c) 2009 Cisco and/or its affiliates.
+ * Copyright (c) 2009-2019 Cisco and/or its affiliates.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at:
@@ -51,6 +51,8 @@ svm_queue_init (void *base, int nels, int elsize)
     clib_unix_warning ("mutexattr_init");
   if (pthread_mutexattr_setpshared (&attr, PTHREAD_PROCESS_SHARED))
     clib_unix_warning ("pthread_mutexattr_setpshared");
+  if (pthread_mutexattr_setrobust (&attr, PTHREAD_MUTEX_ROBUST))
+    clib_unix_warning ("setrobust");
   if (pthread_mutex_init (&q->mutex, &attr))
     clib_unix_warning ("mutex_init");
   if (pthread_mutexattr_destroy (&attr))
@@ -96,7 +98,18 @@ svm_queue_free (svm_queue_t * q)
 void
 svm_queue_lock (svm_queue_t * q)
 {
-  pthread_mutex_lock (&q->mutex);
+  int rv = pthread_mutex_lock (&q->mutex);
+  if (PREDICT_FALSE (rv == EOWNERDEAD))
+    pthread_mutex_consistent (&q->mutex);
+}
+
+static int
+svm_queue_trylock (svm_queue_t * q)
+{
+  int rv = pthread_mutex_trylock (&q->mutex);
+  if (PREDICT_FALSE (rv == EOWNERDEAD))
+    rv = pthread_mutex_consistent (&q->mutex);
+  return rv;
 }
 
 void
@@ -112,7 +125,7 @@ svm_queue_is_full (svm_queue_t * q)
 }
 
 static inline void
-svm_queue_send_signal (svm_queue_t * q, u8 is_prod)
+svm_queue_send_signal_inline (svm_queue_t * q, u8 is_prod)
 {
   if (q->producer_evtfd == -1)
     {
@@ -125,9 +138,17 @@ svm_queue_send_signal (svm_queue_t * q, u8 is_prod)
       ASSERT (q->consumer_evtfd > 0 && q->producer_evtfd > 0);
       fd = is_prod ? q->producer_evtfd : q->consumer_evtfd;
       rv = write (fd, &data, sizeof (data));
+      if (PREDICT_FALSE (rv < 0))
+	clib_unix_warning ("signal write on %d returned %d", fd, rv);
     }
 }
 
+void
+svm_queue_send_signal (svm_queue_t * q, u8 is_prod)
+{
+  svm_queue_send_signal_inline (q, is_prod);
+}
+
 static inline void
 svm_queue_wait_inline (svm_queue_t * q)
 {
@@ -140,10 +161,10 @@ svm_queue_wait_inline (svm_queue_t * q)
       /* Fake a wait for event. We could use epoll but that would mean
        * using yet another fd. Should do for now */
       u32 cursize = q->cursize;
-      pthread_mutex_unlock (&q->mutex);
+      svm_queue_unlock (q);
       while (q->cursize == cursize)
 	CLIB_PAUSE ();
-      pthread_mutex_lock (&q->mutex);
+      svm_queue_lock (q);
     }
 }
 
@@ -170,11 +191,11 @@ svm_queue_timedwait_inline (svm_queue_t * q, double timeout)
       u32 cursize = q->cursize;
       int rv;
 
-      pthread_mutex_unlock (&q->mutex);
+      svm_queue_unlock (q);
       while (q->cursize == cursize && unix_time_now () < max_time)
 	CLIB_PAUSE ();
       rv = unix_time_now () < max_time ? 0 : ETIMEDOUT;
-      pthread_mutex_lock (&q->mutex);
+      svm_queue_lock (q);
       return rv;
     }
 }
@@ -201,7 +222,7 @@ svm_queue_add_nolock (svm_queue_t * q, u8 * elem)
     }
 
   tailp = (i8 *) (&q->data[0] + q->elsize * q->tail);
-  clib_memcpy (tailp, elem, q->elsize);
+  clib_memcpy_fast (tailp, elem, q->elsize);
 
   q->tail++;
   q->cursize++;
@@ -212,7 +233,7 @@ svm_queue_add_nolock (svm_queue_t * q, u8 * elem)
     q->tail = 0;
 
   if (need_broadcast)
-    svm_queue_send_signal (q, 1);
+    svm_queue_send_signal_inline (q, 1);
   return 0;
 }
 
@@ -222,13 +243,13 @@ svm_queue_add_raw (svm_queue_t * q, u8 * elem)
   i8 *tailp;
 
   tailp = (i8 *) (&q->data[0] + q->elsize * q->tail);
-  clib_memcpy (tailp, elem, q->elsize);
+  clib_memcpy_fast (tailp, elem, q->elsize);
 
   q->tail = (q->tail + 1) % q->maxsize;
   q->cursize++;
 
   if (q->cursize == 1)
-    svm_queue_send_signal (q, 1);
+    svm_queue_send_signal_inline (q, 1);
 }
 
 
@@ -244,19 +265,19 @@ svm_queue_add (svm_queue_t * q, u8 * elem, int nowait)
   if (nowait)
     {
       /* zero on success */
-      if (pthread_mutex_trylock (&q->mutex))
+      if (svm_queue_trylock (q))
 	{
 	  return (-1);
 	}
     }
   else
-    pthread_mutex_lock (&q->mutex);
+    svm_queue_lock (q);
 
   if (PREDICT_FALSE (q->cursize == q->maxsize))
     {
       if (nowait)
 	{
-	  pthread_mutex_unlock (&q->mutex);
+	  svm_queue_unlock (q);
 	  return (-2);
 	}
       while (q->cursize == q->maxsize)
@@ -264,7 +285,7 @@ svm_queue_add (svm_queue_t * q, u8 * elem, int nowait)
     }
 
   tailp = (i8 *) (&q->data[0] + q->elsize * q->tail);
-  clib_memcpy (tailp, elem, q->elsize);
+  clib_memcpy_fast (tailp, elem, q->elsize);
 
   q->tail++;
   q->cursize++;
@@ -275,9 +296,9 @@ svm_queue_add (svm_queue_t * q, u8 * elem, int nowait)
     q->tail = 0;
 
   if (need_broadcast)
-    svm_queue_send_signal (q, 1);
+    svm_queue_send_signal_inline (q, 1);
 
-  pthread_mutex_unlock (&q->mutex);
+  svm_queue_unlock (q);
 
   return 0;
 }
@@ -294,19 +315,19 @@ svm_queue_add2 (svm_queue_t * q, u8 * elem, u8 * elem2, int nowait)
   if (nowait)
     {
       /* zero on success */
-      if (pthread_mutex_trylock (&q->mutex))
+      if (svm_queue_trylock (q))
 	{
 	  return (-1);
 	}
     }
   else
-    pthread_mutex_lock (&q->mutex);
+    svm_queue_lock (q);
 
   if (PREDICT_FALSE (q->cursize + 1 == q->maxsize))
     {
       if (nowait)
 	{
-	  pthread_mutex_unlock (&q->mutex);
+	  svm_queue_unlock (q);
 	  return (-2);
 	}
       while (q->cursize + 1 == q->maxsize)
@@ -314,7 +335,7 @@ svm_queue_add2 (svm_queue_t * q, u8 * elem, u8 * elem2, int nowait)
     }
 
   tailp = (i8 *) (&q->data[0] + q->elsize * q->tail);
-  clib_memcpy (tailp, elem, q->elsize);
+  clib_memcpy_fast (tailp, elem, q->elsize);
 
   q->tail++;
   q->cursize++;
@@ -325,7 +346,7 @@ svm_queue_add2 (svm_queue_t * q, u8 * elem, u8 * elem2, int nowait)
   need_broadcast = (q->cursize == 1);
 
   tailp = (i8 *) (&q->data[0] + q->elsize * q->tail);
-  clib_memcpy (tailp, elem2, q->elsize);
+  clib_memcpy_fast (tailp, elem2, q->elsize);
 
   q->tail++;
   q->cursize++;
@@ -334,9 +355,9 @@ svm_queue_add2 (svm_queue_t * q, u8 * elem, u8 * elem2, int nowait)
     q->tail = 0;
 
   if (need_broadcast)
-    svm_queue_send_signal (q, 1);
+    svm_queue_send_signal_inline (q, 1);
 
-  pthread_mutex_unlock (&q->mutex);
+  svm_queue_unlock (q);
 
   return 0;
 }
@@ -355,19 +376,19 @@ svm_queue_sub (svm_queue_t * q, u8 * elem, svm_q_conditional_wait_t cond,
   if (cond == SVM_Q_NOWAIT)
     {
       /* zero on success */
-      if (pthread_mutex_trylock (&q->mutex))
+      if (svm_queue_trylock (q))
 	{
 	  return (-1);
 	}
     }
   else
-    pthread_mutex_lock (&q->mutex);
+    svm_queue_lock (q);
 
   if (PREDICT_FALSE (q->cursize == 0))
     {
       if (cond == SVM_Q_NOWAIT)
 	{
-	  pthread_mutex_unlock (&q->mutex);
+	  svm_queue_unlock (q);
 	  return (-2);
 	}
       else if (cond == SVM_Q_TIMEDWAIT)
@@ -377,7 +398,7 @@ svm_queue_sub (svm_queue_t * q, u8 * elem, svm_q_conditional_wait_t cond,
 
 	  if (rc == ETIMEDOUT)
 	    {
-	      pthread_mutex_unlock (&q->mutex);
+	      svm_queue_unlock (q);
 	      return ETIMEDOUT;
 	    }
 	}
@@ -389,7 +410,7 @@ svm_queue_sub (svm_queue_t * q, u8 * elem, svm_q_conditional_wait_t cond,
     }
 
   headp = (i8 *) (&q->data[0] + q->elsize * q->head);
-  clib_memcpy (elem, headp, q->elsize);
+  clib_memcpy_fast (elem, headp, q->elsize);
 
   q->head++;
   /* $$$$ JFC shouldn't this be == 0? */
@@ -402,9 +423,9 @@ svm_queue_sub (svm_queue_t * q, u8 * elem, svm_q_conditional_wait_t cond,
     q->head = 0;
 
   if (need_broadcast)
-    svm_queue_send_signal (q, 0);
+    svm_queue_send_signal_inline (q, 0);
 
-  pthread_mutex_unlock (&q->mutex);
+  svm_queue_unlock (q);
 
   return 0;
 }
@@ -415,15 +436,15 @@ svm_queue_sub2 (svm_queue_t * q, u8 * elem)
   int need_broadcast;
   i8 *headp;
 
-  pthread_mutex_lock (&q->mutex);
+  svm_queue_lock (q);
   if (q->cursize == 0)
     {
-      pthread_mutex_unlock (&q->mutex);
+      svm_queue_unlock (q);
       return -1;
     }
 
   headp = (i8 *) (&q->data[0] + q->elsize * q->head);
-  clib_memcpy (elem, headp, q->elsize);
+  clib_memcpy_fast (elem, headp, q->elsize);
 
   q->head++;
   need_broadcast = (q->cursize == q->maxsize / 2);
@@ -431,10 +452,10 @@ svm_queue_sub2 (svm_queue_t * q, u8 * elem)
 
   if (PREDICT_FALSE (q->head == q->maxsize))
     q->head = 0;
-  pthread_mutex_unlock (&q->mutex);
+  svm_queue_unlock (q);
 
   if (need_broadcast)
-    svm_queue_send_signal (q, 0);
+    svm_queue_send_signal_inline (q, 0);
 
   return 0;
 }
@@ -442,6 +463,7 @@ svm_queue_sub2 (svm_queue_t * q, u8 * elem)
 int
 svm_queue_sub_raw (svm_queue_t * q, u8 * elem)
 {
+  int need_broadcast;
   i8 *headp;
 
   if (PREDICT_FALSE (q->cursize == 0))
@@ -451,11 +473,16 @@ svm_queue_sub_raw (svm_queue_t * q, u8 * elem)
     }
 
   headp = (i8 *) (&q->data[0] + q->elsize * q->head);
-  clib_memcpy (elem, headp, q->elsize);
+  clib_memcpy_fast (elem, headp, q->elsize);
+
+  need_broadcast = q->cursize == q->maxsize;
 
   q->head = (q->head + 1) % q->maxsize;
   q->cursize--;
 
+  if (PREDICT_FALSE (need_broadcast))
+    svm_queue_send_signal_inline (q, 0);
+
   return 0;
 }