{ "apidoc": { "version": "0.9.6" }, "name": "svc-sundial", "description": "A service for scheduling and running Docker-based jobs", "unions": { "process_schedule": { "description": "A specification for when a process should be run", "types": [ { "type": "cron_schedule" }, { "type": "continuous_schedule" } ] }, "task_executable": { "types": [ { "type": "docker_image_command", "description": "Docker image to run on ECS with Sundial companion container", "deprecation": { "description": "Running jobs on AWS ECS is now deprecated. Use AWS Batch via batch_image_command instead." } }, { "type": "shell_script_command", "description": "Shell command to run on Sundial service instance (Experimental)" }, { "type": "batch_image_command", "description": "Docker image to run on AWS Batch" }, { "type": "emr_command", "description": "Command to submit Spark Jobs on AWS EMR" } ] }, "notification": { "types": [ { "type": "email" }, { "type": "pagerduty" } ] }, "emr_cluster": { "description": "Whether or not to create a new EMR cluster or re-use a pre-existing one", "types": [ { "type": "new_emr_cluster" }, { "type": "existing_emr_cluster" } ] }, "emr_service_role": { "description": "The EMR service role to use", "types": [ { "type": "default_emr_service_role" }, { "type": "custom_emr_service_role" } ] }, "emr_job_flow_role": { "description": "The EMR role to use on the EC2 instances", "types": [ { "type": "default_emr_job_flow_role" }, { "type": "custom_emr_job_flow_role" } ] }, "aws_market": { "description": "The EC2 market of the EC2 instances", "types": [ { "type": "on_demand" }, { "type": "spot" } ] } }, "enums": { "task_status": { "values": [ { "name": "submitted", "description": "The task has been submitted" }, { "name": "runnable" }, { "name": "starting" }, { "name": "pending", "description": "The task is waiting on compute resources" }, { "name": "running", "description": "The task is currently executing or awaiting backoff" }, { "name": "failed", "description": "The task has irrevocably failed" }, { "name": "succeeded", "description": "The task has succeeded without serious errors" } ] }, "process_status": { "values": [ { "name": "running", "description": "The process has tasks currently executing" }, { "name": "succeeded", "description": "All of the process's tasks succeeded on its last run" }, { "name": "failed", "description": "At least one of the process's tasks failed on its last run" } ] }, "process_overlap_action": { "values": [ { "name": "wait", "description": "The process should wait until the currently running instance finishes" }, { "name": "terminate", "description": "The currently running process should be killed" } ] }, "notification_options": { "values": [ { "name": "always", "description": "Always notify when a process completes" }, { "name": "on_failure", "description": "Notify when a process fails" }, { "name": "on_state_change", "description": "Notify when a process goes from succeeding to failing and vica versa" }, { "name": "on_state_change_and_failures", "description": "Notify when going from failing to succeeded and on each failure" }, { "name": "never", "description": "Never notify" } ] }, "emr_release_label": { "description": "Version (aka label in AWS-EMR) of the EMR stack to create.", "values": [ { "name": "emr-5.14.0" }, { "name": "emr-5.13.0" }, { "name": "emr-5.12.1" }, { "name": "emr-5.12.0" }, { "name": "emr-5.11.1" }, { "name": "emr-5.11.0" }, { "name": "emr-5.10.0" }, { "name": "emr-5.9.0" }, { "name": "emr-4.9.2" }, { "name": "emr-4.9.1" } ] }, "emr_application": { "description": "List of applications to install in the EMR cluster", "values": [ { "name": "Hadoop" }, { "name": "Hive" }, { "name": "Mahout" }, { "name": "Pig" }, { "name": "Spark" } ] }, "default_emr_service_role": { "description": "The default EMR Service role", "values": [ { "value": "EMR_DefaultRole", "name": "default_emr_service_role" } ] }, "default_emr_job_flow_role": { "description": "The default EMR EC2 Instance role", "values": [ { "value": "EMR_EC2_DefaultRole", "name": "default_emr_job_flow_role" } ] }, "on_demand": { "description": "The onDemand type of Market Type", "values": [ { "name": "on_demand" } ] } }, "models": { "email": { "description": "An email to send notifications to", "fields": [ { "name": "name", "type": "string", "required": true }, { "name": "email", "type": "string", "required": true }, { "name": "notify_when", "type": "notification_options", "default": "on_state_change_and_failures" } ] }, "pagerduty": { "description": "Pager Duty integration", "fields": [ { "name": "service_key", "type": "string", "required": true }, { "name": "num_consecutive_failures", "type": "integer", "required": true, "default": 1 }, { "name": "api_url", "type": "string", "required": true, "default": "https://events.pagerduty.com" } ] }, "process_definition": { "description": "A grouping of related tasks that are run as a single unit on the same schedule", "fields": [ { "name": "process_definition_name", "type": "string", "required": true }, { "name": "paused", "description": "If true, ignore schedule and only start process if triggered manually", "type": "boolean", "required": false }, { "name": "process_description", "type": "string", "required": false }, { "name": "schedule", "description": "The schedule that the process runs on; if not specified, the process will only run when triggered manually", "type": "process_schedule", "required": false }, { "name": "task_definitions", "type": "[task_definition]", "required": true }, { "name": "overlap_action", "type": "process_overlap_action", "required": true, "default": "wait" }, { "name": "notifications", "type": "[notification]", "required": false } ] }, "task_definition": { "description": "An individual task that runs as part of a process", "fields": [ { "name": "task_definition_name", "type": "string", "description": "The canonical name for this task used by other tasks to identify this task", "required": true }, { "name": "dependencies", "type": "[task_dependency]", "description": "The tasks that must have completed prior to this one beginning", "required": true }, { "name": "executable", "type": "task_executable", "required": true }, { "name": "max_attempts", "type": "integer", "required": true }, { "name": "max_runtime_seconds", "type": "integer", "description": "The execution time (for a single attempt) after which the system will kill the task", "required": false }, { "name": "backoff_base_seconds", "type": "integer", "required": true }, { "name": "backoff_exponent", "type": "double", "required": true, "default": 1.0 }, { "name": "require_explicit_success", "description": "If true, the task must explicitly update its status with Sundial in order to succeed.", "type": "boolean", "default": false } ] }, "task_dependency": { "fields": [ { "name": "task_definition_name", "type": "string", "required": true }, { "name": "success_required", "type": "boolean", "required": true, "default": true } ] }, "batch_image_command": { "fields": [ { "name": "image", "type": "string", "required": true, "description": "Name of docker image including registry URL if needed" }, { "name": "tag", "type": "string", "required": true, "default": "latest", "description": "Tag on docker image" }, { "name": "command", "type": "[string]", "required": true, "description": "Command to pass to Docker container" }, { "name": "memory", "type": "integer", "required": true }, { "name": "vCpus", "type": "integer" }, { "name": "job_role_arn", "type": "string", "description": "ARN of an IAM role, see http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html", "required": false }, { "name": "environment_variables", "type": "[environment_variable]", "required": true, "default": [], "description": "environment variables to be passed to container" }, { "name": "job_queue", "type": "string", "required": false, "description": "Override default job queue, eg: for priority queue or GPU instances queue" } ] }, "docker_image_command": { "fields": [ { "name": "image", "type": "string", "required": true }, { "name": "tag", "type": "string", "required": true, "default": "latest" }, { "name": "command", "type": "[string]", "required": true }, { "name": "memory", "type": "integer", "required": false }, { "name": "cpu", "type": "integer", "required": false }, { "name": "taskRoleArn", "type": "string", "required": false }, { "name": "log_paths", "type": "[string]", "required": true, "default": [] }, { "name": "environment_variables", "type": "[environment_variable]", "required": true, "default": [] } ] }, "shell_script_command": { "fields": [ { "name": "script", "type": "string", "required": true }, { "name": "environment_variables", "type": "[environment_variable]", "required": false } ] }, "emr_instance_group_details": { "description": "Additional configuration for Master/Core/Task EMR instance Groups.", "fields": [ { "name": "emr_instance_type", "type": "string" }, { "name": "instance_count", "type": "integer" }, { "name": "aws_market", "type": "aws_market" }, { "name": "ebs_volume_size", "type": "integer", "description": "optional EBS volume in GB to attach to the emr instance, for simplicity only one volume for the given size and type gp2 will be created.", "required": false } ] }, "new_emr_cluster": { "description": "New EMR cluster configuration object", "fields": [ { "name": "name", "type": "string" }, { "name": "release_label", "type": "emr_release_label", "default": "emr-5.10.0" }, { "name": "applications", "type": "[emr_application]" }, { "name": "s3_log_uri", "type": "string" }, { "name": "master_instance", "type": "emr_instance_group_details" }, { "name": "core_instance", "type": "emr_instance_group_details", "required": false }, { "name": "task_instance", "type": "emr_instance_group_details", "required": false }, { "name": "ec2_subnet", "type": "string", "required": false, "description": "To be set in case the EMR cannot be launched in the `default VPC`" }, { "name": "emr_service_role", "type": "emr_service_role", "description": "role to be assigned to the service, the default one should cover most of the cases" }, { "name": "emr_job_flow_role", "type": "emr_job_flow_role", "description": "role to be assigned to the ec2 instances composing the EMR cluster. The default one should cover most of the cases" }, { "name": "visible_to_all_users", "type": "boolean", "default": "false", "description": "whether or not to show the new cluster in the list of clusters. For more details see [emr docs](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-emr-cluster.html#cfn-emr-cluster-visibletoallusers)" } ] }, "existing_emr_cluster": { "description": "Existing EMR cluster configuration object", "fields": [ { "name": "cluster_id", "type": "string", "description": "The EMR cluster id", "example": "j-2CRR69WTM7N31" } ] }, "s3_cp": { "description": "Wrapper of the S3DistCp spark job for more info visit the aws [s3-dist-cp](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/UsingEMR_s3distcp.html) page", "fields": [ { "name": "source", "type": "string", "description": "Source folder or file to copy. It can be either a s3 or HDFS location", "example": "s3:///my-bucket-root - hdfs://tmp/my-temp-folder" }, { "name": "destination", "type": "string", "description": "Source folder or file to copy. It can be either a s3 or HDFS location", "example": "s3:///my-bucket-root - hdfs://tmp/my-temp-folder" } ] }, "emr_command": { "fields": [ { "name": "emr_cluster", "type": "emr_cluster", "required": true, "description": "The EMR cluster on which to launch the job (aka step)" }, { "name": "job_name", "type": "string", "required": true, "description": "NAme to be assigned to the Job" }, { "name": "region", "type": "string", "required": true, "description": "the AWS Region", "default": "us-east-1" }, { "name": "class", "type": "string", "description": "The spark job's `Main.class` to run", "example": "org.apache.spark.examples.SparkPi" }, { "name": "s3_jar_path", "type": "string", "required": true, "description": "the s3 path to the jar of the job to be run", "example": "s3://my-emr-source-bucket/my-emr-jar.jar" }, { "name": "spark_conf", "type": "[string]", "example": "[\"spark.driver.extraJavaOptions=-Denvironment=integration\"]", "description": "options that will be sent to spark as --conf key=value, e.g. `--conf spark.driver.extraJavaOptions=-Denvironment=integration`" }, { "name": "args", "type": "[string]", "required": true, "description": "command line arguments to be passe to the job's main class", "example": "[\"--executor-memory\", \"1g\"]" }, { "name": "s3_log_details", "description": "AWS EMR periodically (currently every 5 minutes) zips up all the logs and upload them into S3. This does not work well with sundial's _live logs_. Make use of S3 Logs to visualise in real time job's log on sundial's live logs panel", "type": { "model": "s3_log_details", "fields": [ { "name": "log_group_name", "type": "string" }, { "name": "log_stream_name", "type": "string" } ] }, "required": false }, { "name": "load_data", "description": "will trigger a s3 dist cp job for every entry in this array, can be used to load data from s3 into HDFS. **requires Hadoop** as application on the cluster", "type": "[s3_cp]", "required": false }, { "name": "save_results", "description": "will trigger a s3 dist cp job for every entry in this array, can be used to save data from HDFS to s3. **requires Hadoop** as application on the cluster", "type": "[s3_cp]", "required": false } ] }, "custom_emr_service_role": { "description": "Custom, preconfigured, EMR Service role", "fields": [ { "name": "role_name", "type": "string" } ] }, "custom_emr_job_flow_role": { "description": "Custom, preconfigured, EMR EC2 instance role", "fields": [ { "name": "role_name", "type": "string" } ] }, "spot": { "description": "the maximum spot bid price", "fields": [ { "name": "bid_price", "type": "decimal" } ] }, "environment_variable": { "fields": [ { "name": "variable_name", "type": "string", "required": true }, { "name": "value", "type": "string", "required": true } ] }, "cron_schedule": { "description": "See http://quartz-scheduler.org/api/2.2.0/org/quartz/CronExpression.html", "fields": [ { "name": "day_of_week", "type": "string", "required": true }, { "name": "month", "type": "string", "required": true }, { "name": "day_of_month", "type": "string", "required": true }, { "name": "hours", "type": "string", "required": true }, { "name": "minutes", "type": "string", "required": true } ] }, "continuous_schedule": { "fields": [ { "name": "buffer_seconds", "type": "integer", "required": false, "description": "The minimum amount of time (in seconds) that must pass between executions of the process " } ] }, "process": { "fields": [ { "name": "process_id", "type": "uuid", "required": true }, { "name": "process_definition_name", "type": "string", "required": true }, { "name": "start_time", "type": "date-time-iso8601", "required": true }, { "name": "status", "type": "process_status", "required": true }, { "name": "task", "type": "[task]", "required": true } ] }, "task": { "fields": [ { "name": "task_id", "type": "uuid", "required": true }, { "name": "process_id", "type": "uuid", "required": true }, { "name": "process_definition_name", "type": "string", "required": true }, { "name": "task_definition_name", "type": "string", "required": true }, { "name": "start_time", "type": "date-time-iso8601", "required": true }, { "name": "end_time", "type": "date-time-iso8601", "required": false }, { "name": "previous_attempt_count", "type": "integer", "required": true }, { "name": "log_entries", "type": "[log_entry]", "required": true }, { "name": "metadata_entries", "type": "[metadata_entry]", "required": true }, { "name": "execution_state", "type": "[metadata_entry]", "description": "Internal bookkeeping metadata used for task scheduling (e.g. ECS task ID and cluster name)", "required": false }, { "name": "status", "type": "task_status", "required": true } ] }, "log_entry": { "fields": [ { "name": "log_entry_id", "type": "uuid", "required": true, "description": "Uniquely identifies the log message to prevent duplication" }, { "name": "when", "type": "date-time-iso8601", "required": true }, { "name": "source", "type": "string", "required": true }, { "name": "message", "type": "string", "required": true } ] }, "metadata_entry": { "fields": [ { "name": "metadata_entry_id", "type": "uuid", "required": true, "description": "Uniquely identifies the metadata entry to prevent duplication" }, { "name": "when", "type": "date-time-iso8601", "required": true }, { "name": "key", "type": "string", "required": true }, { "name": "value", "type": "string", "required": true } ] }, "healthcheck": { "fields": [ { "name": "status", "type": "string" } ] } }, "resources": { "process_definition": { "path": "/api/process_definitions", "operations": [ { "method": "GET", "parameters": [], "path": "/", "description": "Get all currently registered process definitions", "responses": { "200": { "type": "[process_definition]" } } }, { "method": "GET", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" } ], "path": "/:process_definition_name", "description": "Get a registered process definition", "responses": { "200": { "type": "process_definition" } } }, { "method": "PUT", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" } ], "body": { "type": "process_definition" }, "path": "/:process_definition_name", "description": "Updates or creates a process definition", "responses": { "200": { "type": "process_definition" } } }, { "method": "DELETE", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" } ], "path": "/:process_definition_name", "description": "Deletes a registered process definition", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" }, { "name": "task_definition_name", "location": "query", "required": false, "type": "string" } ], "path": "/:process_definition_name/trigger", "description": "Triggers a new instance of the process", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" } ], "path": "/:process_definition_name/pause", "description": "Pause the process schedule", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" } ], "path": "/:process_definition_name/resume", "description": "Resume the process schedule", "responses": { "204": { "type": "unit" } } } ] }, "process": { "path": "/api/processes", "operations": [ { "method": "GET", "parameters": [ { "name": "process_definition_name", "required": false, "type": "string" }, { "name": "start_time", "required": false, "type": "date-time-iso8601" }, { "name": "end_time", "required": false, "type": "date-time-iso8601" }, { "name": "max_records", "required": false, "type": "integer" }, { "name": "allowed_statuses", "required": true, "type": "[process_status]" } ], "path": "/", "description": "Retrieves a list of process runs based on search parameters.", "responses": { "200": { "type": "[process]" } } }, { "method": "GET", "parameters": [ { "name": "process_id", "required": true, "type": "uuid" } ], "path": "/:process_id", "description": "Retrieves a process run", "responses": { "200": { "type": "process" } } }, { "method": "POST", "parameters": [ { "name": "process_id", "required": true, "type": "uuid" } ], "path": "/:process_id/retry", "description": "Retries a failed process run by restarting failed tasks; tasks will be given a single additional attempt", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "process_id", "required": true, "type": "uuid" } ], "path": "/:process_id/kill", "description": "Terminates an active process", "responses": { "204": { "type": "unit" } } } ] }, "task": { "path": "/api/tasks", "operations": [ { "method": "GET", "parameters": [ { "name": "process_definition_name", "required": true, "type": "string" }, { "name": "task_definition_name", "required": true, "type": "string" }, { "name": "allowed_statuses", "required": true, "type": "[task_status]" }, { "name": "start_time", "required": false, "type": "date-time-iso8601" }, { "name": "end_time", "required": false, "type": "date-time-iso8601" }, { "name": "limit", "required": false, "type": "integer" } ], "path": "/", "description": "Retrieves the most recent tasks meeting the given criteria", "responses": { "200": { "type": "[task]" } } }, { "method": "POST", "parameters": [ { "name": "task_id", "required": true, "type": "uuid" } ], "body": { "type": "[log_entry]" }, "path": "/:task_id/log_entries", "description": "Appends log entries for a task; intended for use within the task executable", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "task_id", "required": true, "type": "uuid" } ], "body": { "type": "[metadata_entry]" }, "path": "/:task_id/metadata", "description": "Appends metadata entries for a task; intended for use within the task executable", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "task_id", "required": true, "type": "uuid" } ], "path": "/:task_id/succeed", "description": "Marks the task as having succeeded", "responses": { "204": { "type": "unit" } } }, { "method": "POST", "parameters": [ { "name": "task_id", "required": true, "type": "uuid" } ], "path": "/:task_id/fail", "description": "Marks the task as having failed", "responses": { "204": { "type": "unit" } } } ] }, "healthcheck": { "path": "/_internal_", "operations": [ { "method": "GET", "path": "/metrics", "responses": { "200": { "type": "string" } } } ] } } }