feat(etl): optimization

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: I2f313d5967e1ac745b817c4074487ccf56940b0a

diff --git a/csit.infra.etl/coverage_rls2206.py b/csit.infra.etl/coverage_rls2206.py
index 4df7898..4e2619d 100644 (file)
--- a/csit.infra.etl/coverage_rls2206.py
+++ b/csit.infra.etl/coverage_rls2206.py
@@ -145,7 +145,6 @@ filtered_paths = [path for path in paths if "report-coverage-2206" in path]
 
 for schema_name in ["mrr", "ndrpdr", "soak", "device"]:
     out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
-    out_sdf.show(truncate=False)
     out_sdf.printSchema()
     out_sdf = out_sdf \
         .withColumn("year", lit(datetime.now().year)) \

diff --git a/csit.infra.etl/iterative_rls2206.py b/csit.infra.etl/iterative_rls2206.py
index f4625cf..88c644b 100644 (file)
--- a/csit.infra.etl/iterative_rls2206.py
+++ b/csit.infra.etl/iterative_rls2206.py
@@ -145,7 +145,6 @@ filtered_paths = [path for path in paths if "report-iterative-2206" in path]
 
 for schema_name in ["mrr", "ndrpdr", "soak"]:
     out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
-    out_sdf.show(truncate=False)
     out_sdf.printSchema()
     out_sdf = out_sdf \
         .withColumn("year", lit(datetime.now().year)) \