From 4c6b808980195b3f474607b9412d0917dedac05a Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Wed, 25 May 2022 09:02:29 +0200 Subject: [PATCH] feat(etl): report 2206 Signed-off-by: Peter Mikus Change-Id: I61fc4187ecd9948c2a8091229c5670f9c73e82ae --- csit.infra.etl/coverage_device.json | 12 ++++++++++++ csit.infra.etl/coverage_mrr.json | 12 ++++++++++++ csit.infra.etl/coverage_ndrpdr.json | 12 ++++++++++++ csit.infra.etl/{coverage_rls2202.py => coverage_rls2206.py} | 4 ++-- csit.infra.etl/coverage_soak.json | 12 ++++++++++++ csit.infra.etl/iterative_mrr.json | 12 ++++++++++++ csit.infra.etl/iterative_ndrpdr.json | 12 ++++++++++++ .../{iterative_rls2202.py => iterative_rls2206.py} | 4 ++-- csit.infra.etl/iterative_soak.json | 12 ++++++++++++ 9 files changed, 88 insertions(+), 4 deletions(-) rename csit.infra.etl/{coverage_rls2202.py => coverage_rls2206.py} (99%) rename csit.infra.etl/{iterative_rls2202.py => iterative_rls2206.py} (99%) diff --git a/csit.infra.etl/coverage_device.json b/csit.infra.etl/coverage_device.json index 12989b30fa..2a4327c05f 100644 --- a/csit.infra.etl/coverage_device.json +++ b/csit.infra.etl/coverage_device.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "start_time", diff --git a/csit.infra.etl/coverage_mrr.json b/csit.infra.etl/coverage_mrr.json index 13b80ec82e..531fcfc62f 100644 --- a/csit.infra.etl/coverage_mrr.json +++ b/csit.infra.etl/coverage_mrr.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "result", diff --git a/csit.infra.etl/coverage_ndrpdr.json b/csit.infra.etl/coverage_ndrpdr.json index f188321bfb..37c45820ea 100644 --- a/csit.infra.etl/coverage_ndrpdr.json +++ b/csit.infra.etl/coverage_ndrpdr.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "result", diff --git a/csit.infra.etl/coverage_rls2202.py b/csit.infra.etl/coverage_rls2206.py similarity index 99% rename from csit.infra.etl/coverage_rls2202.py rename to csit.infra.etl/coverage_rls2206.py index 97b0a12b88..4df7898fe1 100644 --- a/csit.infra.etl/coverage_rls2202.py +++ b/csit.infra.etl/coverage_rls2206.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2202" in path] +filtered_paths = [path for path in paths if "report-coverage-2206" in path] for schema_name in ["mrr", "ndrpdr", "soak", "device"]: out_sdf = process_json_to_dataframe(schema_name, filtered_paths) @@ -156,7 +156,7 @@ for schema_name in ["mrr", "ndrpdr", "soak", "device"]: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2202", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2206", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_soak.json b/csit.infra.etl/coverage_soak.json index 59eaec2e9d..5691e8bf5b 100644 --- a/csit.infra.etl/coverage_soak.json +++ b/csit.infra.etl/coverage_soak.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "result", diff --git a/csit.infra.etl/iterative_mrr.json b/csit.infra.etl/iterative_mrr.json index 13b80ec82e..531fcfc62f 100644 --- a/csit.infra.etl/iterative_mrr.json +++ b/csit.infra.etl/iterative_mrr.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "result", diff --git a/csit.infra.etl/iterative_ndrpdr.json b/csit.infra.etl/iterative_ndrpdr.json index f188321bfb..37c45820ea 100644 --- a/csit.infra.etl/iterative_ndrpdr.json +++ b/csit.infra.etl/iterative_ndrpdr.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "result", diff --git a/csit.infra.etl/iterative_rls2202.py b/csit.infra.etl/iterative_rls2206.py similarity index 99% rename from csit.infra.etl/iterative_rls2202.py rename to csit.infra.etl/iterative_rls2206.py index 13b6f4272c..f4625cf9a8 100644 --- a/csit.infra.etl/iterative_rls2202.py +++ b/csit.infra.etl/iterative_rls2206.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2202" in path] +filtered_paths = [path for path in paths if "report-iterative-2206" in path] for schema_name in ["mrr", "ndrpdr", "soak"]: out_sdf = process_json_to_dataframe(schema_name, filtered_paths) @@ -156,7 +156,7 @@ for schema_name in ["mrr", "ndrpdr", "soak"]: try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2202", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2206", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_soak.json b/csit.infra.etl/iterative_soak.json index 59eaec2e9d..5691e8bf5b 100644 --- a/csit.infra.etl/iterative_soak.json +++ b/csit.infra.etl/iterative_soak.json @@ -30,6 +30,18 @@ "nullable": true, "type": "string" }, + { + "metadata": {}, + "name": "tg_type", + "nullable": true, + "type": "string" + }, + { + "metadata": {}, + "name": "tg_version", + "nullable": true, + "type": "string" + }, { "metadata": {}, "name": "result", -- 2.16.6