From 8e577d49173221dec4c4ed9988ada844321cdffe Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Mon, 23 Oct 2023 12:23:33 +0000 Subject: [PATCH] C-Dash: Improve writing pyarrow schemas Change-Id: Icecbbbdd1c872fd6bfd2e0362941b047658c8f9b Signed-off-by: Tibor Frank --- csit.infra.dash/app/cdash/data/data.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py index 783ebe25ff..1633a02062 100644 --- a/csit.infra.dash/app/cdash/data/data.py +++ b/csit.infra.dash/app/cdash/data/data.py @@ -222,12 +222,15 @@ class Data: pd.api.types.is_string_dtype(itm[""]), pd.api.types.is_string_dtype(itm["telemetry"][0]) )): - print(pa.Schema.from_pandas(itm)) + schema = pa.Schema.from_pandas(itm) pa.parquet.write_metadata( - pa.Schema.from_pandas(itm), - f"{C.PATH_TO_SCHEMAS}_tmp_schema" + schema, f"{C.PATH_TO_SCHEMAS}_tmp_schema" ) - print(itm) + logging.info(schema.to_string( + truncate_metadata=False, + show_field_metadata=True, + show_schema_metadata=True + )) break except KeyError: pass -- 2.16.6