2 # The "datacenters" parameter specifies the list of datacenters which should
3 # be considered when placing this task. This must be provided.
4 datacenters = "${datacenters}"
6 # The "type" parameter controls the type of job, which impacts the scheduler's
7 # decision on placement. For a full list of job types and their differences,
8 # please see the online documentation.
10 # https://www.nomadproject.io/docs/jobspec/schedulers
14 # The periodic stanza allows a job to run at fixed times, dates, or intervals.
15 # The easiest way to think about the periodic scheduler is "Nomad cron" or
18 # https://www.nomadproject.io/docs/job-specification/periodic
22 prohibit_overlap = "${prohibit_overlap}"
23 time_zone = "${time_zone}"
26 # The "group" stanza defines a series of tasks that should be co-located on
27 # the same Nomad client. Any task within a group will be placed on the same
30 # https://www.nomadproject.io/docs/job-specification/group
32 group "${job_name}-master" {
33 # The restart stanza configures a tasks's behavior on task failure. Restarts
34 # happen on the client that is running the task.
36 # https://www.nomadproject.io/docs/job-specification/restart
42 # The constraint allows restricting the set of eligible nodes. Constraints
43 # may filter on attributes or client metadata.
45 # For more information and examples on the "volume" stanza, please see
46 # the online documentation at:
48 # https://www.nomadproject.io/docs/job-specification/constraint
51 attribute = "$${attr.cpu.arch}"
57 attribute = "$${node.class}"
61 # The "task" stanza creates an individual unit of work, such as a Docker
62 # container, web application, or batch processing.
64 # https://www.nomadproject.io/docs/job-specification/task.html
66 task "${job_name}-trending" {
67 # The artifact stanza instructs Nomad to fetch and unpack a remote
68 # resource, such as a file, tarball, or binary.
70 # https://www.nomadproject.io/docs/job-specification/artifact
73 source = "git::https://github.com/FDio/csit"
74 destination = "local/csit"
77 # The "driver" parameter specifies the task driver that should be used to
81 # The "config" stanza specifies the driver configuration, which is passed
82 # directly to the driver to start the task. The details of configurations
83 # are specific to each driver, so please see specific driver
84 # documentation for more information.
87 command = "gluesparksubmit"
89 "--driver-memory", "30g",
90 "--executor-memory", "30g",
93 work_dir = "/local/csit/csit.infra.etl"
96 # The env stanza configures a list of environment variables to populate
97 # the task's environment before starting.
99 AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
100 AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
101 AWS_DEFAULT_REGION = "${aws_default_region}"
102 OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
103 OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
104 OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
108 # The "resources" stanza describes the requirements a task needs to
109 # execute. Resource requirements include memory, network, cpu, and more.
110 # This ensures the task will execute on a machine that contains enough
113 # https://www.nomadproject.io/docs/job-specification/resources
120 task "${job_name}-stats" {
121 # The artifact stanza instructs Nomad to fetch and unpack a remote
122 # resource, such as a file, tarball, or binary.
124 # https://www.nomadproject.io/docs/job-specification/artifact
127 source = "git::https://github.com/FDio/csit"
128 destination = "local/csit"
131 # The "driver" parameter specifies the task driver that should be used to
135 # The "config" stanza specifies the driver configuration, which is passed
136 # directly to the driver to start the task. The details of configurations
137 # are specific to each driver, so please see specific driver
138 # documentation for more information.
141 command = "gluesparksubmit"
143 "--driver-memory", "10g",
144 "--executor-memory", "10g",
147 work_dir = "/local/csit/csit.infra.etl"
150 # The env stanza configures a list of environment variables to populate
151 # the task's environment before starting.
153 AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
154 AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
155 AWS_DEFAULT_REGION = "${aws_default_region}"
156 OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
157 OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
158 OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
162 # The "resources" stanza describes the requirements a task needs to
163 # execute. Resource requirements include memory, network, cpu, and more.
164 # This ensures the task will execute on a machine that contains enough
167 # https://www.nomadproject.io/docs/job-specification/resources
175 # group "${job_name}-rls2206" {
176 # # The restart stanza configures a tasks's behavior on task failure. Restarts
177 # # happen on the client that is running the task.
179 # # https://www.nomadproject.io/docs/job-specification/restart
185 # # The constraint allows restricting the set of eligible nodes. Constraints
186 # # may filter on attributes or client metadata.
188 # # For more information and examples on the "volume" stanza, please see
189 # # the online documentation at:
191 # # https://www.nomadproject.io/docs/job-specification/constraint
194 # attribute = "$${attr.cpu.arch}"
200 # attribute = "$${node.class}"
204 # # The "task" stanza creates an individual unit of work, such as a Docker
205 # # container, web application, or batch processing.
207 # # https://www.nomadproject.io/docs/job-specification/task.html
209 # task "${job_name}-coverage" {
210 # # The artifact stanza instructs Nomad to fetch and unpack a remote
211 # # resource, such as a file, tarball, or binary.
213 # # https://www.nomadproject.io/docs/job-specification/artifact
216 # source = "git::https://github.com/FDio/csit"
217 # destination = "local/csit"
220 # # The "driver" parameter specifies the task driver that should be used to
224 # # The "config" stanza specifies the driver configuration, which is passed
225 # # directly to the driver to start the task. The details of configurations
226 # # are specific to each driver, so please see specific driver
227 # # documentation for more information.
230 # command = "gluesparksubmit"
232 # "--driver-memory", "20g",
233 # "--executor-memory", "20g",
234 # "--executor-cores", "2",
235 # "--master", "local[2]",
236 # "coverage_rls2206.py"
238 # work_dir = "/local/csit/csit.infra.etl"
241 # # The env stanza configures a list of environment variables to populate
242 # # the task's environment before starting.
244 # AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
245 # AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
246 # AWS_DEFAULT_REGION = "${aws_default_region}"
247 # OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
248 # OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
249 # OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
253 # # The "resources" stanza describes the requirements a task needs to
254 # # execute. Resource requirements include memory, network, cpu, and more.
255 # # This ensures the task will execute on a machine that contains enough
256 # # resource capacity.
258 # # https://www.nomadproject.io/docs/job-specification/resources
265 # task "${job_name}-iterative" {
266 # # The artifact stanza instructs Nomad to fetch and unpack a remote
267 # # resource, such as a file, tarball, or binary.
269 # # https://www.nomadproject.io/docs/job-specification/artifact
272 # source = "git::https://github.com/FDio/csit"
273 # destination = "local/csit"
276 # # The "driver" parameter specifies the task driver that should be used to
280 # # The "config" stanza specifies the driver configuration, which is passed
281 # # directly to the driver to start the task. The details of configurations
282 # # are specific to each driver, so please see specific driver
283 # # documentation for more information.
286 # command = "gluesparksubmit"
288 # "--driver-memory", "20g",
289 # "--executor-memory", "20g",
290 # "--executor-cores", "2",
291 # "--master", "local[2]",
292 # "iterative_rls2206.py"
294 # work_dir = "/local/csit/csit.infra.etl"
297 # # The env stanza configures a list of environment variables to populate
298 # # the task's environment before starting.
300 # AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
301 # AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
302 # AWS_DEFAULT_REGION = "${aws_default_region}"
303 # OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
304 # OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
305 # OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
309 # # The "resources" stanza describes the requirements a task needs to
310 # # execute. Resource requirements include memory, network, cpu, and more.
311 # # This ensures the task will execute on a machine that contains enough
312 # # resource capacity.
314 # # https://www.nomadproject.io/docs/job-specification/resources