Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: I54e80bc78f6deb5d2774a2ddd7b3b4153ffb106b
23 files changed:
-filtered_paths = [path for path in paths if "report-coverage-2302" in path]
+filtered_paths = [path for path in paths if "report-coverage-2306" in path]
out_sdf = process_json_to_dataframe("device", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("device", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-coverage-2302" in path]
+filtered_paths = [path for path in paths if "report-coverage-2306" in path]
out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
out_sdf.show(truncate=False)
out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
out_sdf.show(truncate=False)
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-coverage-2302" in path]
+filtered_paths = [path for path in paths if "report-coverage-2306" in path]
out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-coverage-2302" in path]
+filtered_paths = [path for path in paths if "report-coverage-2306" in path]
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-coverage-2302" in path]
+filtered_paths = [path for path in paths if "report-coverage-2306" in path]
out_sdf = process_json_to_dataframe("reconf", filtered_paths)
out_sdf.show(truncate=False)
out_sdf = process_json_to_dataframe("reconf", filtered_paths)
out_sdf.show(truncate=False)
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-coverage-2302" in path]
+filtered_paths = [path for path in paths if "report-coverage-2306" in path]
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-iterative-2302" in path]
+filtered_paths = [path for path in paths if "report-iterative-2306" in path]
out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
out_sdf.show(truncate=False)
out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
out_sdf.show(truncate=False)
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-iterative-2302" in path]
+filtered_paths = [path for path in paths if "report-iterative-2306" in path]
out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-iterative-2302" in path]
+filtered_paths = [path for path in paths if "report-iterative-2306" in path]
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-iterative-2302" in path]
+filtered_paths = [path for path in paths if "report-iterative-2306" in path]
out_sdf = process_json_to_dataframe("reconf", filtered_paths)
out_sdf.show(truncate=False)
out_sdf = process_json_to_dataframe("reconf", filtered_paths)
out_sdf.show(truncate=False)
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
-filtered_paths = [path for path in paths if "report-iterative-2302" in path]
+filtered_paths = [path for path in paths if "report-iterative-2306" in path]
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_device_rls2302.py"
+ "coverage_device_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_hoststack_rls2302.py"
+ "coverage_hoststack_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_mrr_rls2302.py"
+ "coverage_mrr_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_reconf_rls2302.py"
+ "coverage_ndrpdr_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_ndrpdr_rls2302.py"
+ "coverage_reconf_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_soak_rls2302.py"
+ "coverage_soak_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_hoststack_rls2302.py"
+ "iterative_hoststack_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_mrr_rls2302.py"
+ "iterative_mrr_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_ndrpdr_rls2302.py"
+ "iterative_ndrpdr_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_reconf_rls2302.py"
+ "iterative_reconf_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_soak_rls2302.py"
+ "iterative_soak_rls2306.py"
]
work_dir = "/local/csit/csit.infra.etl"
}
]
work_dir = "/local/csit/csit.infra.etl"
}
job_name = "etl-trending-ndrpdr"
}
job_name = "etl-trending-ndrpdr"
}
-module "etl-iterative-hoststack-rls2302" {
+module "etl-iterative-hoststack-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-iterative-hoststack-rls2302"
+ job_name = "etl-iterative-hoststack-rls2306"
-module "etl-iterative-mrr-rls2302" {
+module "etl-iterative-mrr-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-iterative-mrr-rls2302"
+ job_name = "etl-iterative-mrr-rls2306"
-module "etl-iterative-ndrpdr-rls2302" {
+module "etl-iterative-ndrpdr-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-iterative-ndrpdr-rls2302"
+ job_name = "etl-iterative-ndrpdr-rls2306"
-module "etl-iterative-reconf-rls2302" {
+module "etl-iterative-reconf-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-iterative-reconf-rls2302"
+ job_name = "etl-iterative-reconf-rls2306"
-module "etl-iterative-soak-rls2302" {
+module "etl-iterative-soak-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-iterative-soak-rls2302"
+ job_name = "etl-iterative-soak-rls2306"
-module "etl-coverage-device-rls2302" {
+module "etl-coverage-device-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-coverage-device-rls2302"
+ job_name = "etl-coverage-device-rls2306"
-module "etl-coverage-hoststack-rls2302" {
+module "etl-coverage-hoststack-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-coverage-hoststack-rls2302"
+ job_name = "etl-coverage-hoststack-rls2306"
-module "etl-coverage-mrr-rls2302" {
+module "etl-coverage-mrr-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-coverage-mrr-rls2302"
+ job_name = "etl-coverage-mrr-rls2306"
-module "etl-coverage-ndrpdr-rls2302" {
+module "etl-coverage-ndrpdr-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-coverage-ndrpdr-rls2302"
+ job_name = "etl-coverage-ndrpdr-rls2306"
-module "etl-coverage-reconf-rls2302" {
+module "etl-coverage-reconf-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-coverage-reconf-rls2302"
+ job_name = "etl-coverage-reconf-rls2306"
-module "etl-coverage-soak-rls2302" {
+module "etl-coverage-soak-rls2306" {
providers = {
nomad = nomad.yul1
}
providers = {
nomad = nomad.yul1
}
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"]
cron = "@daily"
datacenters = ["yul1"]
- job_name = "etl-coverage-soak-rls2302"
+ job_name = "etl-coverage-soak-rls2306"