+++ /dev/null
-{
- "fields": [
- {
- "metadata": {},
- "name": "job",
- "nullable": false,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "build",
- "nullable": false,
- "type": "integer"
- },
- {
- "metadata": {},
- "name": "duration",
- "nullable": true,
- "type": "double"
- },
- {
- "metadata": {},
- "name": "dut_type",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "dut_version",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "tg_type",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "tg_version",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "start_time",
- "nullable": true,
- "type": "timestamp"
- },
- {
- "metadata": {},
- "name": "passed",
- "nullable": true,
- "type": "boolean"
- },
- {
- "metadata": {},
- "name": "test_id",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "test_name_long",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "test_name_short",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "test_type",
- "nullable": true,
- "type": "string"
- },
- {
- "metadata": {},
- "name": "version",
- "nullable": true,
- "type": "string"
- }
- ],
- "type": "struct"
-}
\ No newline at end of file
+++ /dev/null
-#!/usr/bin/env python3
-
-# Copyright (c) 2024 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""ETL script running on top of the s3://"""
-
-from datetime import datetime, timedelta
-from json import load
-from os import environ
-from pytz import utc
-
-import awswrangler as wr
-from awswrangler.exceptions import EmptyDataFrame
-from awsglue.context import GlueContext
-from boto3 import session
-from pyspark.context import SparkContext
-from pyspark.sql.functions import col, lit, regexp_replace
-from pyspark.sql.types import StructType
-
-
-S3_LOGS_BUCKET=environ.get("S3_LOGS_BUCKET", "fdio-logs-s3-cloudfront-index")
-S3_DOCS_BUCKET=environ.get("S3_DOCS_BUCKET", "fdio-docs-s3-cloudfront-index")
-PATH=f"s3://{S3_LOGS_BUCKET}/vex-yul-rot-jenkins-1/csit-vpp-device-*"
-SUFFIX="info.json.gz"
-IGNORE_SUFFIX=[
- "suite.info.json.gz",
- "setup.info.json.gz",
- "teardown.info.json.gz",
- "suite.output.info.json.gz",
- "setup.output.info.json.gz",
- "teardown.output.info.json.gz"
-]
-LAST_MODIFIED_END=utc.localize(
- datetime.strptime(
- f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
- "%Y-%m-%d"
- )
-)
-LAST_MODIFIED_BEGIN=LAST_MODIFIED_END - timedelta(1)
-
-
-def flatten_frame(nested_sdf):
- """Unnest Spark DataFrame in case there nested structered columns.
-
- :param nested_sdf: Spark DataFrame.
- :type nested_sdf: DataFrame
- :returns: Unnest DataFrame.
- :rtype: DataFrame
- """
- stack = [((), nested_sdf)]
- columns = []
- while len(stack) > 0:
- parents, sdf = stack.pop()
- for column_name, column_type in sdf.dtypes:
- if column_type[:6] == "struct":
- projected_sdf = sdf.select(column_name + ".*")
- stack.append((parents + (column_name,), projected_sdf))
- else:
- columns.append(
- col(".".join(parents + (column_name,))) \
- .alias("_".join(parents + (column_name,)))
- )
- return nested_sdf.select(columns)
-
-
-def process_json_to_dataframe(schema_name, paths):
- """Processes JSON to Spark DataFrame.
-
- :param schema_name: Schema name.
- :type schema_name: string
- :param paths: S3 paths to process.
- :type paths: list
- :returns: Spark DataFrame.
- :rtype: DataFrame
- """
- drop_subset = [
- "dut_type", "dut_version",
- "passed",
- "test_name_long", "test_name_short",
- "test_type",
- "version"
- ]
-
- # load schemas
- with open(f"coverage_{schema_name}.json", "r", encoding="UTF-8") as f_schema:
- schema = StructType.fromJson(load(f_schema))
-
- # create empty DF out of schemas
- sdf = spark.createDataFrame([], schema)
-
- # filter list
- filtered = [path for path in paths if schema_name in path]
-
- # select
- for path in filtered:
- print(path)
-
- sdf_loaded = spark \
- .read \
- .option("multiline", "true") \
- .schema(schema) \
- .json(path) \
- .withColumn("job", lit(path.split("/")[4])) \
- .withColumn("build", lit(path.split("/")[5]))
- sdf = sdf.unionByName(sdf_loaded, allowMissingColumns=True)
-
- # drop rows with all nulls and drop rows with null in critical frames
- sdf = sdf.na.drop(how="all")
- sdf = sdf.na.drop(how="any", thresh=None, subset=drop_subset)
-
- # flatten frame
- sdf = flatten_frame(sdf)
-
- return sdf
-
-
-# create SparkContext and GlueContext
-spark_context = SparkContext.getOrCreate()
-spark_context.setLogLevel("WARN")
-glue_context = GlueContext(spark_context)
-spark = glue_context.spark_session
-
-# files of interest
-paths = wr.s3.list_objects(
- path=PATH,
- suffix=SUFFIX,
- last_modified_begin=LAST_MODIFIED_BEGIN,
- last_modified_end=LAST_MODIFIED_END,
- ignore_suffix=IGNORE_SUFFIX,
- ignore_empty=True
-)
-
-filtered_paths = [path for path in paths if "report-coverage-2410" in path]
-
-out_sdf = process_json_to_dataframe("device", filtered_paths)
-out_sdf.printSchema()
-out_sdf = out_sdf \
- .withColumn("year", lit(datetime.now().year)) \
- .withColumn("month", lit(datetime.now().month)) \
- .withColumn("day", lit(datetime.now().day)) \
- .repartition(1)
-
-try:
- boto3_session = session.Session(
- aws_access_key_id=environ["OUT_AWS_ACCESS_KEY_ID"],
- aws_secret_access_key=environ["OUT_AWS_SECRET_ACCESS_KEY"],
- region_name=environ["OUT_AWS_DEFAULT_REGION"]
- )
-except KeyError:
- boto3_session = session.Session()
-
-try:
- wr.s3.to_parquet(
- df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2410",
- dataset=True,
- partition_cols=["test_type", "year", "month", "day"],
- compression="snappy",
- use_threads=True,
- mode="overwrite_partitions",
- boto3_session=boto3_session
- )
-except EmptyDataFrame:
- pass
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-coverage-2410" in path]
+filtered_paths = [path for path in paths if "report-coverage-2502" in path]
out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-coverage-2410" in path]
+filtered_paths = [path for path in paths if "report-coverage-2502" in path]
out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-coverage-2410" in path]
+filtered_paths = [path for path in paths if "report-coverage-2502" in path]
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-coverage-2410" in path]
+filtered_paths = [path for path in paths if "report-coverage-2502" in path]
out_sdf = process_json_to_dataframe("reconf", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-coverage-2410" in path]
+filtered_paths = [path for path in paths if "report-coverage-2502" in path]
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-2410" in path]
+filtered_paths = [path for path in paths if "report-iterative-2502" in path]
out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-2410" in path]
+filtered_paths = [path for path in paths if "report-iterative-2502" in path]
out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-2410" in path]
+filtered_paths = [path for path in paths if "report-iterative-2502" in path]
out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-2410" in path]
+filtered_paths = [path for path in paths if "report-iterative-2502" in path]
out_sdf = process_json_to_dataframe("reconf", filtered_paths)
out_sdf.show(truncate=False)
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-2410" in path]
+filtered_paths = [path for path in paths if "report-iterative-2502" in path]
out_sdf = process_json_to_dataframe("soak", filtered_paths)
out_sdf.printSchema()
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2410",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2502",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
#!/usr/bin/env python3
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#!/usr/bin/env python3
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
+++ /dev/null
-job "${job_name}" {
- datacenters = ["${datacenters}"]
- type = "${type}"
- periodic {
- cron = "${cron}"
- prohibit_overlap = "${prohibit_overlap}"
- time_zone = "${time_zone}"
- }
- group "${job_name}" {
- restart {
- mode = "fail"
- }
- constraint {
- attribute = "$${attr.cpu.arch}"
- operator = "!="
- value = "arm64"
- }
- constraint {
- attribute = "$${node.class}"
- value = "builder"
- }
- task "${job_name}" {
- artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device_rls2410.py"
- destination = "local/"
- }
- artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device.json"
- destination = "local/"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "coverage_device_rls2410.py"
- ]
- work_dir = "/local"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- }
-}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_hoststack_rls2410.py"
+ "coverage_hoststack.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_mrr_rls2410.py"
+ "coverage_mrr.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_ndrpdr_rls2410.py"
+ "coverage_ndrpdr.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_reconf_rls2410.py"
+ "coverage_reconf.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "coverage_soak_rls2410.py"
+ "coverage_soak.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_hoststack_rls2410.py"
+ "iterative_hoststack.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_mrr_rls2410.py"
+ "iterative_mrr.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_ndrpdr_rls2410.py"
+ "iterative_ndrpdr.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_reconf_rls2410.py"
+ "iterative_reconf.py"
]
work_dir = "/local"
}
}
task "${job_name}" {
artifact {
- source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak_rls2410.py"
+ source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak.py"
destination = "local/"
}
artifact {
"--executor-memory", "20g",
"--executor-cores", "2",
"--master", "local[2]",
- "iterative_soak_rls2410.py"
+ "iterative_soak.py"
]
work_dir = "/local"
}
name = "etl/fdio_docs"
}
+data "vault_kv_secret_v2" "csit_docs" {
+ mount = "kv"
+ name = "etl/csit_docs"
+}
+
module "etl" {
for_each = { for job in var.nomad_jobs : job.job_name => job }
providers = {
memory = 50000
},
{
- job_name = "etl-trending-hoststack"
+ job_name = "etl-iterative-hoststack"
memory = 50000
},
{
- job_name = "etl-iterative-hoststack-rls2410"
+ job_name = "etl-iterative-mrr"
memory = 50000
},
{
- job_name = "etl-iterative-mrr-rls2410"
+ job_name = "etl-iterative-ndrpdr"
memory = 50000
},
{
- job_name = "etl-iterative-ndrpdr-rls2410"
+ job_name = "etl-iterative-reconf"
memory = 50000
},
{
- job_name = "etl-iterative-reconf-rls2410"
+ job_name = "etl-iterative-soak"
memory = 50000
},
{
- job_name = "etl-iterative-soak-rls2410"
+ job_name = "etl-coverage-hoststack"
memory = 50000
},
{
- job_name = "etl-coverage-device-rls2410"
+ job_name = "etl-coverage-mrr"
memory = 50000
},
{
- job_name = "etl-coverage-hoststack-rls2410"
+ job_name = "etl-coverage-ndrpdr"
memory = 50000
},
{
- job_name = "etl-coverage-mrr-rls2410"
+ job_name = "etl-coverage-reconf"
memory = 50000
},
{
- job_name = "etl-coverage-ndrpdr-rls2410"
+ job_name = "etl-coverage-soak"
memory = 50000
},
{
- job_name = "etl-coverage-reconf-rls2410"
- memory = 50000
- },
- {
- job_name = "etl-coverage-soak-rls2410"
+ job_name = "etl-trending-hoststack"
memory = 50000
},
{
version = ">= 2.3.0"
}
vault = {
- version = ">= 4.3.0"
+ version = ">= 4.6.0"
}
}
- required_version = ">= 1.5.4"
+ required_version = ">= 1.10.4"
}
Code Review / csit.git / commitdiff