revert(etl): Manual

Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch>
Change-Id: I42ab6a9db31dabcb4928798cb1ef77dd7f37deb5

diff --git a/csit.infra.etl/coverage_device_rls2410.py b/csit.infra.etl/coverage_device_rls2410.py
index 3a0a632..48746da 100644 (file)
--- a/csit.infra.etl/coverage_device_rls2410.py
+++ b/csit.infra.etl/coverage_device_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("device", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/coverage_hoststack_rls2410.py b/csit.infra.etl/coverage_hoststack_rls2410.py
index abed416..080e998 100644 (file)
--- a/csit.infra.etl/coverage_hoststack_rls2410.py
+++ b/csit.infra.etl/coverage_hoststack_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/coverage_mrr_rls2410.py b/csit.infra.etl/coverage_mrr_rls2410.py
index e73cdbb..9400d8d 100644 (file)
--- a/csit.infra.etl/coverage_mrr_rls2410.py
+++ b/csit.infra.etl/coverage_mrr_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/coverage_ndrpdr_rls2410.py b/csit.infra.etl/coverage_ndrpdr_rls2410.py
index d675ede..18b7627 100644 (file)
--- a/csit.infra.etl/coverage_ndrpdr_rls2410.py
+++ b/csit.infra.etl/coverage_ndrpdr_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/coverage_reconf_rls2410.py b/csit.infra.etl/coverage_reconf_rls2410.py
index 2f51eae..e60a3d6 100644 (file)
--- a/csit.infra.etl/coverage_reconf_rls2410.py
+++ b/csit.infra.etl/coverage_reconf_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/coverage_soak_rls2410.py b/csit.infra.etl/coverage_soak_rls2410.py
index d89b11b..5dc9fb7 100644 (file)
--- a/csit.infra.etl/coverage_soak_rls2410.py
+++ b/csit.infra.etl/coverage_soak_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/iterative_hoststack_rls2410.py b/csit.infra.etl/iterative_hoststack_rls2410.py
index 9489d3b..6d9e3d6 100644 (file)
--- a/csit.infra.etl/iterative_hoststack_rls2410.py
+++ b/csit.infra.etl/iterative_hoststack_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/iterative_mrr_rls2410.py b/csit.infra.etl/iterative_mrr_rls2410.py
index 50557fc..d74e6a9 100644 (file)
--- a/csit.infra.etl/iterative_mrr_rls2410.py
+++ b/csit.infra.etl/iterative_mrr_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/iterative_ndrpdr_rls2410.py b/csit.infra.etl/iterative_ndrpdr_rls2410.py
index 5835fa4..0530dc8 100644 (file)
--- a/csit.infra.etl/iterative_ndrpdr_rls2410.py
+++ b/csit.infra.etl/iterative_ndrpdr_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/iterative_reconf_rls2410.py b/csit.infra.etl/iterative_reconf_rls2410.py
index 0781da2..41d2aed 100644 (file)
--- a/csit.infra.etl/iterative_reconf_rls2410.py
+++ b/csit.infra.etl/iterative_reconf_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -154,8 +148,8 @@ out_sdf.show(truncate=False)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try:

diff --git a/csit.infra.etl/iterative_soak_rls2410.py b/csit.infra.etl/iterative_soak_rls2410.py
index 6388b8c..f581c7c 100644 (file)
--- a/csit.infra.etl/iterative_soak_rls2410.py
+++ b/csit.infra.etl/iterative_soak_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
-#LAST_MODIFIED_END=utc.localize(
-#    datetime.strptime(
-#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-#        "%Y-%m-%d"
-#    )
-#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-9-27",
+        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
         "%Y-%m-%d"
     )
 )
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit("9")) \
-    .withColumn("day", lit("27")) \
+    .withColumn("month", lit(datetime.now().month)) \
+    .withColumn("day", lit(datetime.now().day)) \
     .repartition(1)
 
 try: