+++ /dev/null
-#!/scratch/Anaconda2.4.0/bin/python\r
-import pandas as pd\r
-import numpy as np\r
-import matplotlib\r
-\r
-matplotlib.use('Agg')\r
-from matplotlib import pyplot as plt\r
-import os\r
-import time\r
-\r
-\r
-def generate_dframe_for_test(setup_name, test_name, test_data):\r
- test_results = []\r
- test_dates = []\r
- test_build_ids = []\r
- test_mins = set()\r
- test_maxs = set()\r
- for query in test_data:\r
- test_results.append(float(query[5]))\r
- date_formatted = time.strftime("%d-%m-%Y", time.strptime(query[2], "%Y%m%d"))\r
- time_of_res = date_formatted + '-' + query[3] + ':' + query[4]\r
- test_dates.append(time_of_res)\r
- test_build_ids.append(query[8])\r
- test_mins.add(float(query[6]))\r
- test_maxs.add(float(query[7]))\r
- df = pd.DataFrame({test_name: test_results})\r
- df_detailed = pd.DataFrame({(test_name + ' Results'): test_results, (test_name + ' Date'): test_dates,\r
- "Setup": ([setup_name] * len(test_results)), "Build Id": test_build_ids})\r
- stats = tuple([float(df.mean()), min(test_mins), max(test_maxs)]) # stats = (avg_mpps,min,max)\r
- return df, stats, df_detailed\r
-\r
-\r
-def generate_dframe_arr_and_stats_of_tests_per_setup(date, setup_name, setup_dict):\r
- dframe_arr_trend = []\r
- stats_arr = []\r
- dframe_arr_latest = []\r
- dframe_arr_detailed = []\r
- test_names = setup_dict.keys()\r
- for test in test_names:\r
- df, stats, df_detailed = generate_dframe_for_test(setup_name, test, setup_dict[test])\r
- dframe_arr_detailed.append(df_detailed)\r
- dframe_arr_trend.append(df)\r
- stats_arr.append(stats)\r
- df_latest = float(setup_dict[test][-1][5])\r
- dframe_arr_latest.append(df_latest)\r
- dframe_arr_latest = pd.DataFrame({'Date': [date] * len(dframe_arr_latest),\r
- 'Setup': [setup_name],\r
- 'Test Name': test_names,\r
- 'MPPS': dframe_arr_latest},\r
- index=range(1, len(dframe_arr_latest) + 1))\r
- stats_df = pd.DataFrame(stats_arr, index=setup_dict.keys(), columns=['Avg MPPS', 'Golden Min', 'Golden Max'])\r
- stats_df.index.name = 'Test Name'\r
- return dframe_arr_trend, stats_df, dframe_arr_latest, dframe_arr_detailed\r
-\r
-\r
-def create_plot_for_dframe_arr(dframe_arr, setup_name, start_date, end_date, show='no', save_path='',\r
- file_name='_trend_graph'):\r
- dframe_all = pd.concat(dframe_arr, axis=1)\r
- dframe_all = dframe_all.astype(float)\r
- dframe_all.plot()\r
- plt.legend(fontsize='small', loc='best')\r
- plt.ylabel('MPPS/Core (Norm)')\r
- plt.title('Setup: ' + setup_name)\r
- plt.tick_params(\r
- axis='x',\r
- which='both',\r
- bottom='off',\r
- top='off',\r
- labelbottom='off')\r
- plt.xlabel('Time Period: ' + start_date + ' - ' + end_date)\r
- if save_path:\r
- plt.savefig(os.path.join(save_path, setup_name + file_name + '.png'))\r
- if show == 'yes':\r
- plt.show()\r
-\r
-\r
-def create_bar_plot_for_latest_runs_per_setup(dframe_all_tests_latest, setup_name, show='no', save_path=''):\r
- plt.figure()\r
- colors_for_bars = ['b', 'g', 'r', 'c', 'm', 'y']\r
- dframe_all_tests_latest['MPPS'].plot(kind='bar', legend=False, color = colors_for_bars)\r
- dframe_all_tests_latest = dframe_all_tests_latest[['Test Name', 'Setup', 'Date', 'MPPS']]\r
- plt.xticks(rotation='horizontal')\r
- plt.xlabel('Index of Tests')\r
- plt.ylabel('MPPS/Core (Norm)')\r
- plt.title("Test Runs for Setup: " + setup_name)\r
- if save_path:\r
- plt.savefig(os.path.join(save_path, setup_name + '_latest_test_runs.png'))\r
- dframe_all_tests_latest = dframe_all_tests_latest.round(2)\r
- dframe_all_tests_latest.to_csv(os.path.join(save_path, setup_name + '_latest_test_runs_stats.csv'))\r
- if show == 'yes':\r
- plt.show()\r
-\r
-\r
-def create_all_data_per_setup(setup_dict, setup_name, start_date, end_date, show='no', save_path='', add_stats='',\r
- detailed_test_stats=''):\r
- dframe_arr, stats_arr, dframe_latest_arr, dframe_detailed = generate_dframe_arr_and_stats_of_tests_per_setup(\r
- end_date, setup_name,\r
- setup_dict)\r
- if detailed_test_stats:\r
- detailed_table = create_detailed_table(dframe_detailed)\r
- else:\r
- detailed_table = []\r
- create_bar_plot_for_latest_runs_per_setup(dframe_latest_arr, setup_name, show=show, save_path=save_path)\r
- create_plot_for_dframe_arr(dframe_arr, setup_name, start_date, end_date, show, save_path)\r
- if add_stats:\r
- stats_arr = stats_arr.round(2)\r
- stats_arr.to_csv(os.path.join(save_path, setup_name + '_trend_stats.csv'))\r
- plt.close('all')\r
- return detailed_table, dframe_latest_arr\r
-\r
-\r
-def create_detailed_table(dframe_arr_detailed):\r
- result = reduce(lambda x, y: pd.merge(x, y, how='outer'), dframe_arr_detailed)\r
- return result\r
-\r
-\r
-def latest_runs_comparison_bar_chart(setup_name1, setup_name2, setup1_latest_result, setup2_latest_result, save_path='',\r
- show='no'):\r
- s1_res = setup1_latest_result[['Test Name', 'MPPS']]\r
- s2_res = setup2_latest_result[['Test Name', 'MPPS','Date']]\r
- s1_res.columns = ['Test Name', setup_name1]\r
- s2_res.columns = ['Test Name', setup_name2, 'Date']\r
- compare_dframe = pd.merge(s1_res, s2_res, on='Test Name')\r
- compare_dframe.plot(kind='bar')\r
- plt.legend(fontsize='small', loc='best')\r
- plt.xticks(rotation='horizontal')\r
- plt.xlabel('Index of Tests')\r
- plt.ylabel('MPPS')\r
- plt.title("Comparison between " + setup_name1 + " and " + setup_name2)\r
- if save_path:\r
- plt.savefig(os.path.join(save_path, "_comparison.png"))\r
- compare_dframe = compare_dframe.round(2)\r
- compare_dframe.to_csv(os.path.join(save_path, '_comparison_stats_table.csv'))\r
- if show == 'yes':\r
- plt.show()\r
-\r
-\r
-# WARNING: if the file _all_stats.csv already exists, this script deletes it, to prevent overflowing of data\r
-# since data is appended to the file\r
-def create_all_data(ga_data, setup_names, start_date, end_date, save_path='', add_stats='', detailed_test_stats=''):\r
- total_detailed_data = []\r
- trex07_latest = []\r
- trex08_latest = []\r
- if detailed_test_stats:\r
- if os.path.exists(os.path.join(save_path, '_detailed_table.csv')):\r
- os.remove(os.path.join(save_path, '_detailed_table.csv'))\r
- for setup_name in setup_names:\r
- if setup_name == 'trex07':\r
- detailed_setup_data, trex07_latest = create_all_data_per_setup(ga_data[setup_name], setup_name, start_date,\r
- end_date,\r
- show='no', save_path=save_path,\r
- add_stats=add_stats,\r
- detailed_test_stats=detailed_test_stats)\r
- elif setup_name == 'trex08':\r
- detailed_setup_data, trex08_latest = create_all_data_per_setup(ga_data[setup_name], setup_name, start_date,\r
- end_date,\r
- show='no', save_path=save_path,\r
- add_stats=add_stats,\r
- detailed_test_stats=detailed_test_stats)\r
- else:\r
- detailed_setup_data = create_all_data_per_setup(ga_data[setup_name], setup_name, start_date, end_date,\r
- show='no', save_path=save_path,\r
- add_stats=add_stats,\r
- detailed_test_stats=detailed_test_stats)[0]\r
- total_detailed_data.append(detailed_setup_data)\r
- if detailed_test_stats:\r
- total_detailed_dframe = pd.DataFrame().append(total_detailed_data)\r
- total_detailed_dframe.to_csv(os.path.join(save_path, '_detailed_table.csv'))\r
- latest_runs_comparison_bar_chart('Mellanox ConnectX-4',\r
- 'Intel XL710', trex07_latest, trex08_latest,\r
- save_path=save_path, show='no')\r
Code Review / trex.git / commitdiff