Code Review
/
csit.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
feat(etl): 2306
[csit.git]
/
csit.infra.etl
/
iterative_ndrpdr_rls2306.py
diff --git
a/csit.infra.etl/iterative_ndrpdr_rls2302.py
b/csit.infra.etl/iterative_ndrpdr_rls2306.py
similarity index 99%
rename from
csit.infra.etl/iterative_ndrpdr_rls2302.py
rename to
csit.infra.etl/iterative_ndrpdr_rls2306.py
index
81e6f48
..
9b3a36b
100644
(file)
--- a/
csit.infra.etl/iterative_ndrpdr_rls2302.py
+++ b/
csit.infra.etl/iterative_ndrpdr_rls2306.py
@@
-141,7
+141,7
@@
paths = wr.s3.list_objects(
ignore_empty=True
)
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-230
2
" in path]
+filtered_paths = [path for path in paths if "report-iterative-230
6
" in path]
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
@@
-154,7
+154,7
@@
out_sdf = out_sdf \
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls230
2
",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls230
6
",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",