2 # The "datacenters" parameter specifies the list of datacenters which should
3 # be considered when placing this task. This must be provided.
4 datacenters = "${datacenters}"
6 # The "type" parameter controls the type of job, which impacts the scheduler's
7 # decision on placement. For a full list of job types and their differences,
8 # please see the online documentation.
10 # https://www.nomadproject.io/docs/jobspec/schedulers
14 # The periodic stanza allows a job to run at fixed times, dates, or intervals.
15 # The easiest way to think about the periodic scheduler is "Nomad cron" or
18 # https://www.nomadproject.io/docs/job-specification/periodic
22 prohibit_overlap = "${prohibit_overlap}"
23 time_zone = "${time_zone}"
26 # The "group" stanza defines a series of tasks that should be co-located on
27 # the same Nomad client. Any task within a group will be placed on the same
30 # https://www.nomadproject.io/docs/job-specification/group
32 group "${job_name}-master" {
33 # The restart stanza configures a tasks's behavior on task failure. Restarts
34 # happen on the client that is running the task.
36 # https://www.nomadproject.io/docs/job-specification/restart
42 # The constraint allows restricting the set of eligible nodes. Constraints
43 # may filter on attributes or client metadata.
45 # For more information and examples on the "volume" stanza, please see
46 # the online documentation at:
48 # https://www.nomadproject.io/docs/job-specification/constraint
51 attribute = "$${attr.cpu.arch}"
57 attribute = "$${node.class}"
61 # The "task" stanza creates an individual unit of work, such as a Docker
62 # container, web application, or batch processing.
64 # https://www.nomadproject.io/docs/job-specification/task.html
66 task "${job_name}-trending" {
67 # The artifact stanza instructs Nomad to fetch and unpack a remote
68 # resource, such as a file, tarball, or binary.
70 # https://www.nomadproject.io/docs/job-specification/artifact
73 source = "git::https://github.com/FDio/csit"
74 destination = "local/csit"
77 # The "driver" parameter specifies the task driver that should be used to
81 # The "config" stanza specifies the driver configuration, which is passed
82 # directly to the driver to start the task. The details of configurations
83 # are specific to each driver, so please see specific driver
84 # documentation for more information.
87 command = "gluesparksubmit"
89 "--driver-memory", "20g",
90 "--executor-memory", "20g",
93 work_dir = "/local/csit/csit.infra.etl"
96 # The env stanza configures a list of environment variables to populate
97 # the task's environment before starting.
99 AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
100 AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
101 AWS_DEFAULT_REGION = "${aws_default_region}"
102 OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
103 OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
104 OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
108 # The "resources" stanza describes the requirements a task needs to
109 # execute. Resource requirements include memory, network, cpu, and more.
110 # This ensures the task will execute on a machine that contains enough
113 # https://www.nomadproject.io/docs/job-specification/resources
120 task "${job_name}-stats" {
121 # The artifact stanza instructs Nomad to fetch and unpack a remote
122 # resource, such as a file, tarball, or binary.
124 # https://www.nomadproject.io/docs/job-specification/artifact
127 source = "git::https://github.com/FDio/csit"
128 destination = "local/csit"
131 # The "driver" parameter specifies the task driver that should be used to
135 # The "config" stanza specifies the driver configuration, which is passed
136 # directly to the driver to start the task. The details of configurations
137 # are specific to each driver, so please see specific driver
138 # documentation for more information.
141 command = "gluesparksubmit"
143 "--driver-memory", "10g",
144 "--executor-memory", "10g",
147 work_dir = "/local/csit/csit.infra.etl"
150 # The env stanza configures a list of environment variables to populate
151 # the task's environment before starting.
153 AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
154 AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
155 AWS_DEFAULT_REGION = "${aws_default_region}"
156 OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
157 OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
158 OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
162 # The "resources" stanza describes the requirements a task needs to
163 # execute. Resource requirements include memory, network, cpu, and more.
164 # This ensures the task will execute on a machine that contains enough
167 # https://www.nomadproject.io/docs/job-specification/resources
175 group "${job_name}-rls2206" {
176 # The restart stanza configures a tasks's behavior on task failure. Restarts
177 # happen on the client that is running the task.
179 # https://www.nomadproject.io/docs/job-specification/restart
185 # The constraint allows restricting the set of eligible nodes. Constraints
186 # may filter on attributes or client metadata.
188 # For more information and examples on the "volume" stanza, please see
189 # the online documentation at:
191 # https://www.nomadproject.io/docs/job-specification/constraint
194 attribute = "$${attr.cpu.arch}"
200 attribute = "$${node.class}"
204 # The "task" stanza creates an individual unit of work, such as a Docker
205 # container, web application, or batch processing.
207 # https://www.nomadproject.io/docs/job-specification/task.html
209 task "${job_name}-coverage" {
210 # The artifact stanza instructs Nomad to fetch and unpack a remote
211 # resource, such as a file, tarball, or binary.
213 # https://www.nomadproject.io/docs/job-specification/artifact
216 source = "git::https://github.com/FDio/csit"
217 destination = "local/csit"
220 # The "driver" parameter specifies the task driver that should be used to
224 # The "config" stanza specifies the driver configuration, which is passed
225 # directly to the driver to start the task. The details of configurations
226 # are specific to each driver, so please see specific driver
227 # documentation for more information.
230 command = "gluesparksubmit"
232 "--driver-memory", "20g",
233 "--executor-memory", "20g",
234 "--executor-cores", "2",
235 "--master", "local[2]",
236 "coverage_rls2202.py"
238 work_dir = "/local/csit/csit.infra.etl"
241 # The env stanza configures a list of environment variables to populate
242 # the task's environment before starting.
244 AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
245 AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
246 AWS_DEFAULT_REGION = "${aws_default_region}"
247 OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
248 OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
249 OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
253 # The "resources" stanza describes the requirements a task needs to
254 # execute. Resource requirements include memory, network, cpu, and more.
255 # This ensures the task will execute on a machine that contains enough
258 # https://www.nomadproject.io/docs/job-specification/resources
265 task "${job_name}-iterative" {
266 # The artifact stanza instructs Nomad to fetch and unpack a remote
267 # resource, such as a file, tarball, or binary.
269 # https://www.nomadproject.io/docs/job-specification/artifact
272 source = "git::https://github.com/FDio/csit"
273 destination = "local/csit"
276 # The "driver" parameter specifies the task driver that should be used to
280 # The "config" stanza specifies the driver configuration, which is passed
281 # directly to the driver to start the task. The details of configurations
282 # are specific to each driver, so please see specific driver
283 # documentation for more information.
286 command = "gluesparksubmit"
288 "--driver-memory", "20g",
289 "--executor-memory", "20g",
290 "--executor-cores", "2",
291 "--master", "local[2]",
292 "iterative_rls2206.py"
294 work_dir = "/local/csit/csit.infra.etl"
297 # The env stanza configures a list of environment variables to populate
298 # the task's environment before starting.
300 AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
301 AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
302 AWS_DEFAULT_REGION = "${aws_default_region}"
303 OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
304 OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
305 OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
309 # The "resources" stanza describes the requirements a task needs to
310 # execute. Resource requirements include memory, network, cpu, and more.
311 # This ensures the task will execute on a machine that contains enough
314 # https://www.nomadproject.io/docs/job-specification/resources