fixed issue with fetching elk data - now fetching from trex-perf-* (with wildcard)

Signed-off-by: itraviv <[email protected]>

diff --git a/doc/AnalyticsWebReport.py b/doc/AnalyticsWebReport.py
index ed35d86..89c7771 100755 (executable)
--- a/doc/AnalyticsWebReport.py
+++ b/doc/AnalyticsWebReport.py
@@ -18,7 +18,7 @@ def main(verbose=False, source='elk', detailed_test_stats='yes'):
         response = ac.get_report(analytics, start_date, current_date)
         all_data_dict, setups = ac.export_to_tuples(response)
     if source == 'elk':
-        elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-000004', port=9200)
+        elk_manager = ec.ELKManager(hostname='sceasr-b20', index='trex_perf-*', port=9200)
         all_data_dict = elk_manager.fetch_and_parse()
     dest_path = os.path.join(os.getcwd(), 'build', 'images')
     if verbose:

diff --git a/doc/ELKConnect.py b/doc/ELKConnect.py
index fb7e2c2..fe3e121 100755 (executable)
--- a/doc/ELKConnect.py
+++ b/doc/ELKConnect.py
@@ -18,7 +18,7 @@ import elasticsearch.helpers
 \r
 \r
 class ELKManager:\r
-    def __init__(self, hostname, index='trex_perf-000004', port=9200):\r
+    def __init__(self, hostname, index='trex_perf-*', port=9200):\r
         self.hostname = hostname\r
         self.index = index\r
         self.port = port\r
@@ -29,14 +29,14 @@ class ELKManager:
 \r
     @staticmethod\r
     def time_res_calculation():\r
-        milli_since_epoch = int(time.time() * 1000)\r
+        seconds_since_epoch = int(time.time())\r
         time_2w_ago = datetime.date.timetuple(datetime.datetime.utcnow() - datetime.timedelta(weeks=2))\r
-        two_w_ago_epoch_milli = int(time.mktime(time_2w_ago) * 1000)\r
-        return milli_since_epoch, two_w_ago_epoch_milli\r
+        two_w_ago_epoch_second = int(time.mktime(time_2w_ago))\r
+        return seconds_since_epoch, two_w_ago_epoch_second\r
 \r
     def fetch_all_data(self):\r
         res = {}\r
-        milli_since_epoch, two_weeks_ago_epoch_milli = self.time_res_calculation()\r
+        seconds_since_epoch, two_w_ago_epoch_second = self.time_res_calculation()\r
         for setup_name in self.setup_names:\r
             query = {\r
                 "_source": ["info.setup.name", "test.name", "test.mpps_pc", "timestamp", "build_id"],\r
@@ -44,15 +44,15 @@ class ELKManager:
                 "query": {\r
                     "bool": {\r
                         "filter": [\r
-                            {"range": {\r
-                                "timestamp": {"gte": two_weeks_ago_epoch_milli, "lte": milli_since_epoch,\r
-                                              "format": "epoch_millis"}}},\r
                             {"term": {"info.setup.name": setup_name}},\r
-                            {"term": {"test.type": "stateless"}}\r
+                            {"term": {"test.type": "stateless"}},\r
+                            {"range": {"timestamp": {"gte": two_w_ago_epoch_second, "lte": seconds_since_epoch,\r
+                                                     "format": "epoch_second"}}}\r
                         ]\r
                     }\r
                 }\r
             }\r
+\r
             res[setup_name] = list(elasticsearch.helpers.scan(self.es, index=self.index, query=query, size=10000))\r
         self.all_data_raw = res\r
 \r

diff --git a/doc/TRexDataAnalysis.py b/doc/TRexDataAnalysis.py
index 5a2966c..9aeba0e 100755 (executable)
--- a/doc/TRexDataAnalysis.py
+++ b/doc/TRexDataAnalysis.py
@@ -225,8 +225,9 @@ def create_all_data(ga_data, end_date, save_path='', detailed_test_stats=''):
     if detailed_test_stats:
         if os.path.exists(os.path.join(save_path, '_detailed_table.csv')):
             os.remove(os.path.join(save_path, '_detailed_table.csv'))
-        all_setups_data_dframe = pd.DataFrame().append(all_setups_data)
-        all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))
+        if all_setups_data:
+            all_setups_data_dframe = pd.DataFrame().append(all_setups_data)
+            all_setups_data_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))
 
     trex07setup = all_setups['trex07']
     trex08setup = all_setups['trex08']