C-Dash: Add rls2502 data

Change-Id: I58c0aecc7e598d95b6d2e7e52d357b9526cd6fd0
Signed-off-by: Tibor Frank <[email protected]>

diff --git a/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2502_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2502_ndrpdr
new file mode 100644 (file)
index 0000000..06bc618
Binary files /dev/null and b/csit.infra.dash/app/cdash/data/_metadata/coverage_rls2502_ndrpdr differ

diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_hoststack b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_hoststack
new file mode 100644 (file)
index 0000000..1e9c708
Binary files /dev/null and b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_hoststack differ

diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_mrr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_mrr
new file mode 100644 (file)
index 0000000..ced7896
Binary files /dev/null and b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_mrr differ

diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_ndrpdr b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_ndrpdr
new file mode 100644 (file)
index 0000000..cf2b8a1
Binary files /dev/null and b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_ndrpdr differ

diff --git a/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_soak b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_soak
new file mode 100644 (file)
index 0000000..4502ca4
Binary files /dev/null and b/csit.infra.dash/app/cdash/data/_metadata/iterative_rls2502_soak differ

diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py
index 41033a7..749769c 100644 (file)
--- a/csit.infra.dash/app/cdash/data/data.py
+++ b/csit.infra.dash/app/cdash/data/data.py
@@ -14,6 +14,7 @@
 """Prepare data for Plotly Dash applications.
 """
 
+import botocore
 import logging
 import resource
 import awswrangler as wr
@@ -286,6 +287,9 @@ class Data:
         if days:
             last_modified_begin = datetime.now(tz=UTC) - timedelta(days=days)
         try:
+            wr.config.botocore_config = botocore.config.Config(
+                max_pool_connections=C.MAX_POOL_SIZE
+            )
             df = wr.s3.read_parquet(
                 path=path,
                 path_suffix="parquet",

diff --git a/csit.infra.dash/app/cdash/data/data.yaml b/csit.infra.dash/app/cdash/data/data.yaml
index c1aa0bf..d807f6a 100644 (file)
--- a/csit.infra.dash/app/cdash/data/data.yaml
+++ b/csit.infra.dash/app/cdash/data/data.yaml
@@ -436,3 +436,150 @@
     - passed
     - test_id
     - version
+- data_type: iterative
+  partition: test_type
+  partition_name: mrr
+  release: rls2502
+  path: s3://csit-docs-s3-cloudfront-index/csit/parquet/iterative_rls2502
+  schema: iterative_rls2502_mrr
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - tg_type
+    - hosts
+    - start_time
+    - passed
+    - test_id
+    - version
+    - result_receive_rate_rate_avg
+    - result_receive_rate_rate_stdev
+    - result_receive_rate_rate_unit
+    - result_receive_rate_bandwidth_avg
+    - result_receive_rate_bandwidth_stdev
+    - result_receive_rate_bandwidth_unit
+- data_type: iterative
+  partition: test_type
+  partition_name: ndrpdr
+  release: rls2502
+  path: s3://csit-docs-s3-cloudfront-index/csit/parquet/iterative_rls2502
+  schema: iterative_rls2502_ndrpdr
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - tg_type
+    - hosts
+    - start_time
+    - passed
+    - test_id
+    - version
+    - result_pdr_lower_rate_unit
+    - result_pdr_lower_rate_value
+    - result_ndr_lower_rate_unit
+    - result_ndr_lower_rate_value
+    - result_pdr_lower_bandwidth_unit
+    - result_pdr_lower_bandwidth_value
+    - result_ndr_lower_bandwidth_unit
+    - result_ndr_lower_bandwidth_value
+    - result_latency_reverse_pdr_90_hdrh
+    - result_latency_reverse_pdr_50_hdrh
+    - result_latency_reverse_pdr_10_hdrh
+    - result_latency_reverse_pdr_0_hdrh
+    - result_latency_forward_pdr_90_hdrh
+    - result_latency_forward_pdr_50_avg
+    - result_latency_forward_pdr_50_hdrh
+    - result_latency_forward_pdr_50_unit
+    - result_latency_forward_pdr_10_hdrh
+    - result_latency_forward_pdr_0_hdrh
+- data_type: iterative
+  partition: test_type
+  partition_name: hoststack
+  release: rls2502
+  path: s3://csit-docs-s3-cloudfront-index/csit/parquet/iterative_rls2502
+  schema: iterative_rls2502_hoststack
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - hosts
+    - tg_type
+    - result_bandwidth_unit
+    - result_bandwidth_value
+    - result_rate_unit
+    - result_rate_value
+    - start_time
+    - passed
+    - test_id
+    - version
+- data_type: iterative
+  partition: test_type
+  partition_name: soak
+  release: rls2502
+  path: s3://csit-docs-s3-cloudfront-index/csit/parquet/iterative_rls2502
+  schema: iterative_rls2502_soak
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - hosts
+    - tg_type
+    - result_critical_rate_lower_bandwidth_unit
+    - result_critical_rate_lower_bandwidth_value
+    - result_critical_rate_lower_rate_unit
+    - result_critical_rate_lower_rate_value
+    - start_time
+    - passed
+    - telemetry
+    - test_id
+    - version
+- data_type: coverage
+  partition: test_type
+  partition_name: ndrpdr
+  release: rls2502
+  path: s3://csit-docs-s3-cloudfront-index/csit/parquet/coverage_rls2502
+  schema: coverage_rls2502_ndrpdr
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - tg_type
+    - start_time
+    - passed
+    - test_id
+    - version
+    - result_pdr_lower_rate_unit
+    - result_pdr_lower_rate_value
+    - result_ndr_lower_rate_value
+    - result_pdr_lower_bandwidth_value
+    - result_ndr_lower_bandwidth_value
+    - result_latency_reverse_pdr_90_hdrh
+    - result_latency_reverse_pdr_50_hdrh
+    - result_latency_reverse_pdr_10_hdrh
+    - result_latency_forward_pdr_90_hdrh
+    - result_latency_forward_pdr_50_hdrh
+    - result_latency_forward_pdr_10_hdrh
+- data_type: coverage
+  partition: test_type
+  partition_name: mrr
+  release: rls2502
+  path: s3://csit-docs-s3-cloudfront-index/csit/parquet/coverage_rls2502
+  schema: iterative_rls2502_mrr
+  columns:
+    - job
+    - build
+    - dut_type
+    - dut_version
+    - tg_type
+    - start_time
+    - passed
+    - test_id
+    - version
+    - result_receive_rate_rate_avg
+    - result_receive_rate_rate_stdev
+    - result_receive_rate_rate_unit

diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index d17f060..823ecc6 100644 (file)
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -154,6 +154,9 @@ class Constants:
     # The file with tooltips.
     TOOLTIP_FILE = "cdash/utils/tooltips.yaml"
 
+    # Max pool size for boto3
+    MAX_POOL_SIZE = get_int_from_env("MAX_POOL_SIZE", 30)
+
     # Maximal value of TIME_PERIOD for data read from the parquets in days.
     # Do not change without a good reason.
     MAX_TIME_PERIOD = 250