From a4324a996f34902579338033110d27575a654c8e Mon Sep 17 00:00:00 2001
From: Dave Barach <dave@barachs.net>
Date: Tue, 19 Feb 2019 17:05:30 -0500
Subject: [PATCH] calculate per-thread time offset

The main thread squirrels away vlib_time_now (&vlib_global_main),
worker threads use it to calculate an offset in f64 seconds from their
own vlib_time_now(vm) value. We use that offset until the next barrier
sync.

Thanks to Damjan for the suggestion.

Change-Id: If56cdfe68e5ad8ac3b0d0fc885dc3ba556cd1215
Signed-off-by: Dave Barach <dave@barachs.net>
---
 src/vlib/main.h    |  5 ++++-
 src/vlib/threads.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 src/vlib/threads.h | 15 +++++++++++++++
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/src/vlib/main.h b/src/vlib/main.h
index f89ecd3299f..4192c3f2720 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -63,6 +63,9 @@ typedef struct vlib_main_t
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
   /* Instruction level timing state. */
   clib_time_t clib_time;
+  /* Offset from main thread time */
+  f64 time_offset;
+  f64 time_last_barrier_release;
 
   /* Time stamp of last node dispatch. */
   u64 cpu_time_last_node_dispatch;
@@ -232,7 +235,7 @@ void vlib_worker_loop (vlib_main_t * vm);
 always_inline f64
 vlib_time_now (vlib_main_t * vm)
 {
-  return clib_time_now (&vm->clib_time);
+  return clib_time_now (&vm->clib_time) + vm->time_offset;
 }
 
 always_inline f64
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index cdb4eb082b5..4368a74ac9f 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -1510,6 +1510,14 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
 
   deadline = now + BARRIER_SYNC_TIMEOUT;
 
+  /*
+   * Note when we let go of the barrier.
+   * Workers can use this to derive a reasonably accurate
+   * time offset. See vlib_time_now(...)
+   */
+  vm->time_last_barrier_release = vlib_time_now (vm);
+  CLIB_MEMORY_STORE_BARRIER ();
+
   *vlib_worker_threads->wait_at_barrier = 0;
 
   while (*vlib_worker_threads->workers_at_barrier > 0)
@@ -1808,6 +1816,45 @@ threads_init (vlib_main_t * vm)
 
 VLIB_INIT_FUNCTION (threads_init);
 
+
+static clib_error_t *
+show_clock_command_fn (vlib_main_t * vm,
+		       unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  int i;
+  f64 now;
+
+  now = vlib_time_now (vm);
+
+  vlib_cli_output (vm, "Time now %.9f", now);
+
+  if (vec_len (vlib_mains) == 1)
+    return 0;
+
+  vlib_cli_output (vm, "Time last barrier release %.9f",
+		   vm->time_last_barrier_release);
+
+  for (i = 1; i < vec_len (vlib_mains); i++)
+    {
+      if (vlib_mains[i] == 0)
+	continue;
+      vlib_cli_output (vm, "Thread %d offset %.9f error %.9f", i,
+		       vlib_mains[i]->time_offset,
+		       vm->time_last_barrier_release -
+		       vlib_mains[i]->time_last_barrier_release);
+    }
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (f_command, static) =
+{
+  .path = "show clock",
+  .short_help = "show clock",
+  .function = show_clock_command_fn,
+};
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index 0e9cba52103..95fcf3d83d7 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -422,6 +422,21 @@ vlib_worker_thread_barrier_check (void)
 	}
       while (*vlib_worker_threads->wait_at_barrier)
 	;
+
+      /*
+       * Recompute the offset from thread-0 time.
+       * Note that vlib_time_now adds vm->time_offset, so
+       * clear it first. Save the resulting idea of "now", to
+       * see how well we're doing. See show_clock_command_fn(...)
+       */
+      {
+	f64 now;
+	vm->time_offset = 0.0;
+	now = vlib_time_now (vm);
+	vm->time_offset = vlib_global_main.time_last_barrier_release - now;
+	vm->time_last_barrier_release = vlib_time_now (vm);
+      }
+
       if (CLIB_DEBUG > 0)
 	vm->parked_at_barrier = 0;
       clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
-- 
2.16.6