Code Review
/
csit.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
feat(terraform): ETL for 2310
[csit.git]
/
csit.infra.etl
/
iterative_soak_rls2310.py
diff --git
a/csit.infra.etl/iterative_soak_rls2306.py
b/csit.infra.etl/iterative_soak_rls2310.py
similarity index 99%
rename from
csit.infra.etl/iterative_soak_rls2306.py
rename to
csit.infra.etl/iterative_soak_rls2310.py
index
ccc5012
..
cfe733b
100644
(file)
--- a/
csit.infra.etl/iterative_soak_rls2306.py
+++ b/
csit.infra.etl/iterative_soak_rls2310.py
@@
-141,7
+141,7
@@
paths = wr.s3.list_objects(
ignore_empty=True
)
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-23
06
" in path]
+filtered_paths = [path for path in paths if "report-iterative-23
10
" in path]
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
@@
-154,7
+154,7
@@
out_sdf = out_sdf \
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls23
06
",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls23
10
",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",