feat(etl): report 2206

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: I61fc4187ecd9948c2a8091229c5670f9c73e82ae

diff --git a/csit.infra.etl/coverage_device.json b/csit.infra.etl/coverage_device.json
index 12989b3..2a4327c 100644 (file)
--- a/csit.infra.etl/coverage_device.json
+++ b/csit.infra.etl/coverage_device.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "start_time",

diff --git a/csit.infra.etl/coverage_mrr.json b/csit.infra.etl/coverage_mrr.json
index 13b80ec..531fcfc 100644 (file)
--- a/csit.infra.etl/coverage_mrr.json
+++ b/csit.infra.etl/coverage_mrr.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "result",

diff --git a/csit.infra.etl/coverage_ndrpdr.json b/csit.infra.etl/coverage_ndrpdr.json
index f188321..37c4582 100644 (file)
--- a/csit.infra.etl/coverage_ndrpdr.json
+++ b/csit.infra.etl/coverage_ndrpdr.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "result",

diff --git a/csit.infra.etl/coverage_rls2202.py b/csit.infra.etl/coverage_rls2206.py
similarity index 99%
rename from csit.infra.etl/coverage_rls2202.py
rename to csit.infra.etl/coverage_rls2206.py
index 97b0a12..4df7898 100644 (file)
--- a/csit.infra.etl/coverage_rls2202.py
+++ b/csit.infra.etl/coverage_rls2206.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-coverage-2202" in path]
+filtered_paths = [path for path in paths if "report-coverage-2206" in path]
 
 for schema_name in ["mrr", "ndrpdr", "soak", "device"]:
     out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
@@ -156,7 +156,7 @@ for schema_name in ["mrr", "ndrpdr", "soak", "device"]:
     try:
         wr.s3.to_parquet(
             df=out_sdf.toPandas(),
-            path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2202",
+            path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2206",
             dataset=True,
             partition_cols=["test_type", "year", "month", "day"],
             compression="snappy",

diff --git a/csit.infra.etl/coverage_soak.json b/csit.infra.etl/coverage_soak.json
index 59eaec2..5691e8b 100644 (file)
--- a/csit.infra.etl/coverage_soak.json
+++ b/csit.infra.etl/coverage_soak.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "result",

diff --git a/csit.infra.etl/iterative_mrr.json b/csit.infra.etl/iterative_mrr.json
index 13b80ec..531fcfc 100644 (file)
--- a/csit.infra.etl/iterative_mrr.json
+++ b/csit.infra.etl/iterative_mrr.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "result",

diff --git a/csit.infra.etl/iterative_ndrpdr.json b/csit.infra.etl/iterative_ndrpdr.json
index f188321..37c4582 100644 (file)
--- a/csit.infra.etl/iterative_ndrpdr.json
+++ b/csit.infra.etl/iterative_ndrpdr.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "result",

diff --git a/csit.infra.etl/iterative_rls2202.py b/csit.infra.etl/iterative_rls2206.py
similarity index 99%
rename from csit.infra.etl/iterative_rls2202.py
rename to csit.infra.etl/iterative_rls2206.py
index 13b6f42..f4625cf 100644 (file)
--- a/csit.infra.etl/iterative_rls2202.py
+++ b/csit.infra.etl/iterative_rls2206.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
     ignore_empty=True
 )
 
-filtered_paths = [path for path in paths if "report-iterative-2202" in path]
+filtered_paths = [path for path in paths if "report-iterative-2206" in path]
 
 for schema_name in ["mrr", "ndrpdr", "soak"]:
     out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
@@ -156,7 +156,7 @@ for schema_name in ["mrr", "ndrpdr", "soak"]:
     try:
         wr.s3.to_parquet(
             df=out_sdf.toPandas(),
-            path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2202",
+            path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2206",
             dataset=True,
             partition_cols=["test_type", "year", "month", "day"],
             compression="snappy",

diff --git a/csit.infra.etl/iterative_soak.json b/csit.infra.etl/iterative_soak.json
index 59eaec2..5691e8b 100644 (file)
--- a/csit.infra.etl/iterative_soak.json
+++ b/csit.infra.etl/iterative_soak.json
@@ -30,6 +30,18 @@
             "nullable": true,
             "type": "string"
         },
+        {
+            "metadata": {},
+            "name": "tg_type",
+            "nullable": true,
+            "type": "string"
+        },
+        {
+            "metadata": {},
+            "name": "tg_version",
+            "nullable": true,
+            "type": "string"
+        },
         {
             "metadata": {},
             "name": "result",