Code Review
/
csit.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
feat(etl): report 2206
[csit.git]
/
csit.infra.etl
/
iterative_rls2206.py
diff --git
a/csit.infra.etl/iterative_rls2202.py
b/csit.infra.etl/iterative_rls2206.py
similarity index 99%
rename from
csit.infra.etl/iterative_rls2202.py
rename to
csit.infra.etl/iterative_rls2206.py
index
13b6f42
..
f4625cf
100644
(file)
--- a/
csit.infra.etl/iterative_rls2202.py
+++ b/
csit.infra.etl/iterative_rls2206.py
@@
-141,7
+141,7
@@
paths = wr.s3.list_objects(
ignore_empty=True
)
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-220
2
" in path]
+filtered_paths = [path for path in paths if "report-iterative-220
6
" in path]
for schema_name in ["mrr", "ndrpdr", "soak"]:
out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
for schema_name in ["mrr", "ndrpdr", "soak"]:
out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
@@
-156,7
+156,7
@@
for schema_name in ["mrr", "ndrpdr", "soak"]:
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls220
2
",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls220
6
",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",