openapi: 3.0.3
info:
  title: svc-sundial
  version: 0.2.7
  description: A service for scheduling and running Docker-based jobs
paths:
  /_internal_/metrics:
    get:
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: string
  /api/process_definitions/:
    get:
      description: Get all currently registered process definitions
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsProcessDefinition'
  /api/process_definitions/{process_definition_name}:
    get:
      description: Get a registered process definition
      parameters:
      - name: process_definition_name
        in: path
        required: true
        schema:
          type: string
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsProcessDefinition'
    put:
      description: Updates or creates a process definition
      parameters:
      - name: process_definition_name
        in: path
        required: true
        schema:
          type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsProcessDefinition'
        required: true
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsProcessDefinition'
    delete:
      description: Deletes a registered process definition
      parameters:
      - name: process_definition_name
        in: path
        required: true
        schema:
          type: string
      responses:
        '204':
          description: No content
  /api/process_definitions/{process_definition_name}/pause:
    post:
      description: Pause the process schedule
      parameters:
      - name: process_definition_name
        in: path
        required: true
        schema:
          type: string
      responses:
        '204':
          description: No content
  /api/process_definitions/{process_definition_name}/resume:
    post:
      description: Resume the process schedule
      parameters:
      - name: process_definition_name
        in: path
        required: true
        schema:
          type: string
      responses:
        '204':
          description: No content
  /api/process_definitions/{process_definition_name}/trigger:
    post:
      description: Triggers a new instance of the process
      parameters:
      - name: process_definition_name
        in: path
        required: true
        schema:
          type: string
      - name: task_definition_name
        in: query
        required: false
        schema:
          type: string
      responses:
        '204':
          description: No content
  /api/processes/:
    get:
      description: Retrieves a list of process runs based on search parameters.
      parameters:
      - name: process_definition_name
        in: query
        required: false
        schema:
          type: string
      - name: start_time
        in: query
        required: false
        schema:
          type: string
          format: date-time
      - name: end_time
        in: query
        required: false
        schema:
          type: string
          format: date-time
      - name: max_records
        in: query
        required: false
        schema:
          type: integer
          format: int32
      - name: allowed_statuses
        in: query
        required: true
        schema:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsProcessStatus'
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsProcess'
  /api/processes/{process_id}:
    get:
      description: Retrieves a process run
      parameters:
      - name: process_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsProcess'
  /api/processes/{process_id}/kill:
    post:
      description: Terminates an active process
      parameters:
      - name: process_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      responses:
        '204':
          description: No content
  /api/processes/{process_id}/retry:
    post:
      description: Retries a failed process run by restarting failed tasks; tasks
        will be given a single additional attempt
      parameters:
      - name: process_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      responses:
        '204':
          description: No content
  /api/tasks/:
    get:
      description: Retrieves the most recent tasks meeting the given criteria
      parameters:
      - name: process_definition_name
        in: query
        required: true
        schema:
          type: string
      - name: task_definition_name
        in: query
        required: true
        schema:
          type: string
      - name: allowed_statuses
        in: query
        required: true
        schema:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsTaskStatus'
      - name: start_time
        in: query
        required: false
        schema:
          type: string
          format: date-time
      - name: end_time
        in: query
        required: false
        schema:
          type: string
          format: date-time
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          format: int32
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsTask'
  /api/tasks/{task_id}/fail:
    post:
      description: Marks the task as having failed
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      responses:
        '204':
          description: No content
  /api/tasks/{task_id}/log_entries:
    post:
      description: Appends log entries for a task; intended for use within the task
        executable
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      requestBody:
        content:
          application/json:
            schema:
              type: array
              items:
                $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsLogEntry'
        required: true
      responses:
        '204':
          description: No content
  /api/tasks/{task_id}/metadata:
    post:
      description: Appends metadata entries for a task; intended for use within the
        task executable
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      requestBody:
        content:
          application/json:
            schema:
              type: array
              items:
                $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsMetadataEntry'
        required: true
      responses:
        '204':
          description: No content
  /api/tasks/{task_id}/succeed:
    post:
      description: Marks the task as having succeeded
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
      responses:
        '204':
          description: No content
components:
  schemas:
    ComGiltSvcSundialV0ModelsCustomEmrJobFlowRole:
      description: Custom, preconfigured, EMR EC2 instance role
      type: object
      required:
      - role_name
      properties:
        role_name:
          type: string
    ComGiltSvcSundialV0ModelsTaskDependency:
      type: object
      required:
      - task_definition_name
      - success_required
      properties:
        task_definition_name:
          type: string
        success_required:
          type: boolean
    ComGiltSvcSundialV0ModelsPagerduty:
      description: Pager Duty integration
      type: object
      required:
      - service_key
      - num_consecutive_failures
      - api_url
      properties:
        service_key:
          type: string
        num_consecutive_failures:
          type: integer
          format: int32
        api_url:
          default: https://events.pagerduty.com
          type: string
    ComGiltSvcSundialV0ModelsEnvironmentVariable:
      type: object
      required:
      - variable_name
      - value
      properties:
        variable_name:
          type: string
        value:
          type: string
    ComGiltSvcSundialV0ModelsSpot:
      description: the maximum spot bid price
      type: object
      required:
      - bid_price
      properties:
        bid_price:
          type: number
    ComGiltSvcSundialV0ModelsExistingEmrCluster:
      description: Existing EMR cluster configuration object
      type: object
      required:
      - cluster_id
      properties:
        cluster_id:
          description: The EMR cluster id
          example: j-2CRR69WTM7N31
          type: string
    ComGiltSvcSundialV0ModelsS3LogDetails:
      type: object
      required:
      - log_group_name
      - log_stream_name
      properties:
        log_group_name:
          type: string
        log_stream_name:
          type: string
    ComGiltSvcSundialV0ModelsEmrInstanceGroupDetails:
      description: Additional configuration for Master/Core/Task EMR instance Groups.
      type: object
      required:
      - emr_instance_type
      - instance_count
      - aws_market
      properties:
        emr_instance_type:
          type: string
        instance_count:
          type: integer
          format: int32
        aws_market:
          $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsAwsMarket'
        ebs_volume_size:
          description: optional EBS volume in GB to attach to the emr instance, for
            simplicity only one volume for the given size and type gp2 will be created.
          type: integer
          format: int32
    ComGiltSvcSundialV0ModelsDockerImageCommand:
      type: object
      required:
      - image
      - tag
      - command
      - log_paths
      - environment_variables
      properties:
        image:
          type: string
        tag:
          default: latest
          type: string
        command:
          type: array
          items:
            type: string
        memory:
          type: integer
          format: int32
        cpu:
          type: integer
          format: int32
        taskRoleArn:
          type: string
        log_paths:
          type: array
          items:
            type: string
        environment_variables:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEnvironmentVariable'
    ComGiltSvcSundialV0ModelsContinuousSchedule:
      type: object
      properties:
        buffer_seconds:
          description: The minimum amount of time (in seconds) that must pass between
            executions of the process
          type: integer
          format: int32
    ComGiltSvcSundialV0ModelsEmail:
      description: An email to send notifications to
      type: object
      required:
      - name
      - email
      - notify_when
      properties:
        name:
          type: string
        email:
          type: string
        notify_when:
          $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsNotificationOptions'
    ComGiltSvcSundialV0ModelsCustomEmrServiceRole:
      description: Custom, preconfigured, EMR Service role
      type: object
      required:
      - role_name
      properties:
        role_name:
          type: string
    ComGiltSvcSundialV0ModelsLogEntry:
      type: object
      required:
      - log_entry_id
      - when
      - source
      - message
      properties:
        log_entry_id:
          description: Uniquely identifies the log message to prevent duplication
          type: string
          format: uuid
        when:
          type: string
          format: date-time
        source:
          type: string
        message:
          type: string
    ComGiltSvcSundialV0ModelsBatchImageCommand:
      type: object
      required:
      - image
      - tag
      - command
      - memory
      - vCpus
      - environment_variables
      properties:
        image:
          description: Name of docker image including registry URL if needed
          type: string
        tag:
          description: Tag on docker image
          default: latest
          type: string
        command:
          description: Command to pass to Docker container
          type: array
          items:
            type: string
        memory:
          type: integer
          format: int32
        vCpus:
          type: integer
          format: int32
        job_role_arn:
          description: ARN of an IAM role, see http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
          type: string
        environment_variables:
          description: environment variables to be passed to container
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEnvironmentVariable'
        job_queue:
          description: 'Override default job queue, eg: for priority queue or GPU
            instances queue'
          type: string
    ComGiltSvcSundialV0ModelsCronSchedule:
      description: See http://quartz-scheduler.org/api/2.2.0/org/quartz/CronExpression.html
      type: object
      required:
      - day_of_week
      - month
      - day_of_month
      - hours
      - minutes
      properties:
        day_of_week:
          type: string
        month:
          type: string
        day_of_month:
          type: string
        hours:
          type: string
        minutes:
          type: string
    ComGiltSvcSundialV0ModelsProcessDefinition:
      description: A grouping of related tasks that are run as a single unit on the
        same schedule
      type: object
      required:
      - process_definition_name
      - task_definitions
      - overlap_action
      properties:
        process_definition_name:
          type: string
        paused:
          description: If true, ignore schedule and only start process if triggered
            manually
          type: boolean
        process_description:
          type: string
        schedule:
          $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsProcessSchedule'
          description: The schedule that the process runs on; if not specified, the
            process will only run when triggered manually
        task_definitions:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsTaskDefinition'
        overlap_action:
          $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsProcessOverlapAction'
        notifications:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsNotification'
    ComGiltSvcSundialV0ModelsMetadataEntry:
      type: object
      required:
      - metadata_entry_id
      - when
      - key
      - value
      properties:
        metadata_entry_id:
          description: Uniquely identifies the metadata entry to prevent duplication
          type: string
          format: uuid
        when:
          type: string
          format: date-time
        key:
          type: string
        value:
          type: string
    ComGiltSvcSundialV0ModelsS3Cp:
      description: Wrapper of the S3DistCp spark job for more info visit the aws [s3-dist-cp](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/UsingEMR_s3distcp.html)
        page
      type: object
      required:
      - source
      - destination
      properties:
        source:
          description: Source folder or file to copy. It can be either a s3 or HDFS
            location
          example: s3:///my-bucket-root - hdfs://tmp/my-temp-folder
          type: string
        destination:
          description: Source folder or file to copy. It can be either a s3 or HDFS
            location
          example: s3:///my-bucket-root - hdfs://tmp/my-temp-folder
          type: string
    ComGiltSvcSundialV0ModelsNewEmrCluster:
      description: New EMR cluster configuration object
      type: object
      required:
      - name
      - release_label
      - applications
      - s3_log_uri
      - master_instance
      - emr_service_role
      - emr_job_flow_role
      - visible_to_all_users
      properties:
        name:
          type: string
        release_label:
          $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsEmrReleaseLabel'
        applications:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsEmrApplication'
        s3_log_uri:
          type: string
        master_instance:
          $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEmrInstanceGroupDetails'
        core_instance:
          $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEmrInstanceGroupDetails'
        task_instance:
          $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEmrInstanceGroupDetails'
        ec2_subnet:
          description: To be set in case the EMR cannot be launched in the `default
            VPC`
          type: string
        emr_service_role:
          $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsEmrServiceRole'
          description: role to be assigned to the service, the default one should
            cover most of the cases
        emr_job_flow_role:
          $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsEmrJobFlowRole'
          description: role to be assigned to the ec2 instances composing the EMR
            cluster. The default one should cover most of the cases
        visible_to_all_users:
          description: whether or not to show the new cluster in the list of clusters.
            For more details see [emr docs](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-emr-cluster.html#cfn-emr-cluster-visibletoallusers)
          type: boolean
    ComGiltSvcSundialV0ModelsHealthcheck:
      type: object
      required:
      - status
      properties:
        status:
          type: string
    ComGiltSvcSundialV0ModelsEmrCommand:
      type: object
      required:
      - emr_cluster
      - job_name
      - region
      - class
      - s3_jar_path
      - spark_conf
      - args
      properties:
        emr_cluster:
          $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsEmrCluster'
          description: The EMR cluster on which to launch the job (aka step)
        job_name:
          description: NAme to be assigned to the Job
          type: string
        region:
          description: the AWS Region
          default: us-east-1
          type: string
        class:
          description: The spark job's `Main.class` to run
          example: org.apache.spark.examples.SparkPi
          type: string
        s3_jar_path:
          description: the s3 path to the jar of the job to be run
          example: s3://my-emr-source-bucket/my-emr-jar.jar
          type: string
        spark_conf:
          description: options that will be sent to spark as --conf key=value, e.g.
            `--conf spark.driver.extraJavaOptions=-Denvironment=integration`
          example:
          - spark.driver.extraJavaOptions=-Denvironment=integration
          type: array
          items:
            type: string
        args:
          description: command line arguments to be passe to the job's main class
          example:
          - --executor-memory
          - 1g
          type: array
          items:
            type: string
        s3_log_details:
          $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsS3LogDetails'
          description: AWS EMR periodically (currently every 5 minutes) zips up all
            the logs and upload them into S3. This does not work well with sundial's
            _live logs_. Make use of S3 Logs to visualise in real time job's log on
            sundial's live logs panel
        load_data:
          description: will trigger a s3 dist cp job for every entry in this array,
            can be used to load data from s3 into HDFS. **requires Hadoop** as application
            on the cluster
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsS3Cp'
        save_results:
          description: will trigger a s3 dist cp job for every entry in this array,
            can be used to save data from HDFS to s3. **requires Hadoop** as application
            on the cluster
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsS3Cp'
    ComGiltSvcSundialV0ModelsShellScriptCommand:
      type: object
      required:
      - script
      properties:
        script:
          type: string
        environment_variables:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEnvironmentVariable'
    ComGiltSvcSundialV0ModelsProcess:
      type: object
      required:
      - process_id
      - process_definition_name
      - start_time
      - status
      - task
      properties:
        process_id:
          type: string
          format: uuid
        process_definition_name:
          type: string
        start_time:
          type: string
          format: date-time
        status:
          $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsProcessStatus'
        task:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsTask'
    ComGiltSvcSundialV0ModelsTask:
      type: object
      required:
      - task_id
      - process_id
      - process_definition_name
      - task_definition_name
      - start_time
      - previous_attempt_count
      - log_entries
      - metadata_entries
      - status
      properties:
        task_id:
          type: string
          format: uuid
        process_id:
          type: string
          format: uuid
        process_definition_name:
          type: string
        task_definition_name:
          type: string
        start_time:
          type: string
          format: date-time
        end_time:
          type: string
          format: date-time
        previous_attempt_count:
          type: integer
          format: int32
        log_entries:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsLogEntry'
        metadata_entries:
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsMetadataEntry'
        execution_state:
          description: Internal bookkeeping metadata used for task scheduling (e.g.
            ECS task ID and cluster name)
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsMetadataEntry'
        status:
          $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsTaskStatus'
    ComGiltSvcSundialV0ModelsTaskDefinition:
      description: An individual task that runs as part of a process
      type: object
      required:
      - task_definition_name
      - dependencies
      - executable
      - max_attempts
      - backoff_base_seconds
      - backoff_exponent
      - require_explicit_success
      properties:
        task_definition_name:
          description: The canonical name for this task used by other tasks to identify
            this task
          type: string
        dependencies:
          description: The tasks that must have completed prior to this one beginning
          type: array
          items:
            $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsTaskDependency'
        executable:
          $ref: '#/components/schemas/ComGiltSvcSundialV0UnionsTaskExecutable'
        max_attempts:
          type: integer
          format: int32
        max_runtime_seconds:
          description: The execution time (for a single attempt) after which the system
            will kill the task
          type: integer
          format: int32
        backoff_base_seconds:
          type: integer
          format: int32
        backoff_exponent:
          type: number
          format: double
        require_explicit_success:
          description: If true, the task must explicitly update its status with Sundial
            in order to succeed.
          type: boolean
    ComGiltSvcSundialV0EnumsTaskStatus:
      type: string
      enum:
      - submitted
      - runnable
      - starting
      - pending
      - running
      - failed
      - succeeded
    ComGiltSvcSundialV0EnumsEmrApplication:
      description: List of applications to install in the EMR cluster
      type: string
      enum:
      - Hadoop
      - Hive
      - Mahout
      - Pig
      - Spark
    ComGiltSvcSundialV0EnumsProcessStatus:
      type: string
      enum:
      - running
      - succeeded
      - failed
    ComGiltSvcSundialV0EnumsProcessOverlapAction:
      type: string
      enum:
      - wait
      - terminate
    ComGiltSvcSundialV0EnumsNotificationOptions:
      type: string
      enum:
      - always
      - on_failure
      - on_state_change
      - on_state_change_and_failures
      - never
    ComGiltSvcSundialV0EnumsDefaultEmrJobFlowRole:
      description: The default EMR EC2 Instance role
      type: string
      enum:
      - EMR_EC2_DefaultRole
    ComGiltSvcSundialV0EnumsOnDemand:
      description: The onDemand type of Market Type
      type: string
      enum:
      - on_demand
    ComGiltSvcSundialV0EnumsDefaultEmrServiceRole:
      description: The default EMR Service role
      type: string
      enum:
      - EMR_DefaultRole
    ComGiltSvcSundialV0EnumsEmrReleaseLabel:
      description: Version (aka label in AWS-EMR) of the EMR stack to create.
      type: string
      enum:
      - emr-5.14.0
      - emr-5.13.0
      - emr-5.12.1
      - emr-5.12.0
      - emr-5.11.1
      - emr-5.11.0
      - emr-5.10.0
      - emr-5.9.0
      - emr-4.9.2
      - emr-4.9.1
    ComGiltSvcSundialV0UnionsNotification:
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEmail'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsPagerduty'
    ComGiltSvcSundialV0UnionsEmrCluster:
      description: Whether or not to create a new EMR cluster or re-use a pre-existing
        one
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsNewEmrCluster'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsExistingEmrCluster'
    ComGiltSvcSundialV0UnionsEmrServiceRole:
      description: The EMR service role to use
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsDefaultEmrServiceRole'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsCustomEmrServiceRole'
    ComGiltSvcSundialV0UnionsAwsMarket:
      description: The EC2 market of the EC2 instances
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsOnDemand'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsSpot'
    ComGiltSvcSundialV0UnionsEmrJobFlowRole:
      description: The EMR role to use on the EC2 instances
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0EnumsDefaultEmrJobFlowRole'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsCustomEmrJobFlowRole'
    ComGiltSvcSundialV0UnionsProcessSchedule:
      description: A specification for when a process should be run
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsCronSchedule'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsContinuousSchedule'
    ComGiltSvcSundialV0UnionsTaskExecutable:
      oneOf:
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsDockerImageCommand'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsShellScriptCommand'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsBatchImageCommand'
      - $ref: '#/components/schemas/ComGiltSvcSundialV0ModelsEmrCommand'