jobs.yaml

swagger: '2.0'
info:
  title: Job Manager Service
  description: Job Manager API for interacting with asynchronous batch jobs and workflows.
  license:
    name: BSD
    url: 'http://opensource.org/licenses/BSD-3-Clause'
  version: '0.0.1'
produces:
  - application/json
enum: &TIMEFRAME
  - HOURS_1
  - HOURS_8
  - HOURS_24
  - DAYS_7
  - DAYS_30
  - ALL_TIME

paths:
  '/capabilities':
    get:
      operationId: GetCapabilities
      summary: Get the capabilities of this backend implementation
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/CapabilitiesResponse'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Capabilities

  '/jobs/{id}/abort':
    post:
      operationId: AbortJob
      summary: Abort a job by ID
      parameters:
        - name: id
          description: Job ID
          required: true
          type: string
          in: path
      responses:
        '200':
          description: Success
        '400':
          $ref: '#/responses/BadRequest'
        '401':
          $ref: '#/responses/Unauthorized'
        '404':
          $ref: '#/responses/NotFound'
        '412':
          $ref: '#/responses/JobTerminal'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/jobs/{id}/updateLabels':
    post:
      operationId: UpdateJobLabels
      summary: Update labels on a job.
      parameters:
        - name: id
          description: Job ID
          required: true
          type: string
          in: path
        - name: body
          required: true
          in: body
          schema:
            $ref: '#/definitions/UpdateJobLabelsRequest'
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/UpdateJobLabelsResponse'
        '400':
          $ref: '#/responses/BadRequest'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
        '501':
          description: Server does not implement this method.
      tags:
        - Jobs

  '/jobs/query':
    post:
      # TODO(calbach): Parameterize sort ordering.
      operationId: QueryJobs
      summary: |
        Query jobs by various filter criteria. Returned jobs are ordered from
        newest to oldest submission time.
      parameters:
        - name: body
          required: true
          in: body
          schema:
            $ref: '#/definitions/QueryJobsRequest'
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/QueryJobsResponse'
        '400':
          $ref: '#/responses/BadRequest'
        '401':
          $ref: '#/responses/Unauthorized'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/jobs/{id}':
    get:
      operationId: GetJob
      summary: Query for job and task-level metadata for a specified job
      parameters:
        - name: id
          description: Job ID
          required: true
          type: string
          in: path
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/JobMetadataResponse'
        '400':
          $ref: '#/responses/BadRequest'
        '401':
          $ref: '#/responses/Unauthorized'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/jobs/{id}/{task}/attempts':
    get:
      operationId: GetTaskAttempts
      summary: Query for task-level metadata for a specified job
      parameters:
        - name: id
          description: Job ID
          required: true
          type: string
          in: path
        - name: task
          description: task name
          required: true
          type: string
          in: path
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/JobAttemptsResponse'
        '400':
          $ref: '#/responses/BadRequest'
        '401':
          $ref: '#/responses/Unauthorized'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/jobs/{id}/{task}/{index}/attempts':
    get:
      operationId: GetShardAttempts
      summary: Query for shard-level metadata for a specified job
      parameters:
        - name: id
          description: Job ID
          required: true
          type: string
          in: path
        - name: task
          description: task name
          required: true
          type: string
          in: path
        - name: index
          description: shard index
          required: true
          type: string
          in: path
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/JobAttemptsResponse'
        '400':
          $ref: '#/responses/BadRequest'
        '401':
          $ref: '#/responses/Unauthorized'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/jobs/operationDetails':
    get:
      operationId: GetOperationDetails
      summary: Call for operation details from the Google Pipelines API
      parameters:
        - name: job
          description: job ID
          required: true
          in: query
          type: string
        - name: operation
          description: operation ID
          type: string
          in: query
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/JobOperationResponse'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/jobs/tailFile':
    get:
      operationId: TailFileContents
      summary: Get up to a certain amount (from the end) of content from the Google Storage API
      parameters:
        - name: bucket
          description: Google bucket ID
          required: true
          in: query
          type: string
        - name: object
          description: ID of the file stored in the Google bucket
          required: true
          in: query
          type: string
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/FileContents'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Jobs

  '/aggregations':
    get:
      operationId: GetJobAggregations
      summary: Query for aggregated jobs in the given time frame.
      parameters:
        - name: timeFrame
          description: Time Frame to aggregate over.
          required: true
          in: query
          type: string
          enum: *TIMEFRAME
        - name: projectId
          description: The ID of the project to query.
          type: string
          in: query
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/AggregationResponse'
        '400':
          $ref: '#/responses/BadRequest'
        '401':
          $ref: '#/responses/Unauthorized'
        '404':
          $ref: '#/responses/NotFound'
        '500':
          $ref: '#/responses/ServerError'
      tags:
        - Aggregation

  '/health':
    get:
      operationId: Health
      summary: A health check for the backend and its ability to connect to its underlying backend service
      parameters: []
      responses:
        '200':
          description: Success
          schema:
            $ref: '#/definitions/HealthResponse'
        '503':
          $ref: '#/responses/ServiceUnavailable'
      tags:
        - Jobs

definitions:
  AuthenticationCapability:
    description: Specification of the backend's authentication capabilities
    properties:
      isRequired:
        type: boolean
        description: Whether or not authentication is required
      scopes:
        type: array
        description: OAuth 2.0 requested scopes
        items:
          type: string
      forcedLogoutDomains:
        type: array
        description: Domains where a forced logout will happen after an interval of inactivity has passed.
        items:
          type: string
      forcedLogoutTime:
        type: integer
        description: Number of milliseconds for the interval of inactivity.
      outsideAuth:
        type: boolean
        description: Whether or not to use an outside service for some auth-related functionality.
        default: false

  DisplayField:
    description: Description of a display field
    required:
      - field
      - display
    properties:
      field:
        type: string
      display:
        type: string
      editable:
        type: boolean
        default: false
      bulkEditable:
        type: boolean
        default: false
      fieldType:
        $ref: '#/definitions/FieldType'
      validFieldValues:
        description: A list of values the field can be set to
        type: array
        items:
          type: string
      primary:
        description: >
          Whether this field is of high importance for display purposes.
          For example, a UI with limited space should prioritize displaying
          primary field values.
        type: boolean
        default: true
      filterable:
        description: >
          Whether the user should be able to quickly filter by
          this field in a name:value way.
        type: boolean
        default: false

  CapabilitiesResponse:
    description: Specification of the backend's capabilities
    properties:
      authentication:
        $ref: '#/definitions/AuthenticationCapability'
      displayFields:
        type: array
        description: >
          Fields on QueryJobsResult returned from POST /jobs/query
          populated on some or all jobs. The fields are mapped to their
          display names, in order of importance. Extended fields and
          labels can be included, such as 'label.foo' or
          'extensions.userId'
        items:
          $ref: '#/definitions/DisplayField'
      commonLabels:
        type: array
        description: Common labels which are present on most jobs returned
        items:
          type: string
      queryExtensions:
        type: array
        description: Fields on ExtendedQueryFields which are queryable
        items:
          type: string
      aggregationTimeFrames:
        type: array
        description: Time frames supported by the /aggregations endpoint.
        items:
          $ref: '#/definitions/TimeFrame'

  JobMetadataResponse:
    description: Job and task level metadata
    required:
      - id
      - status
      - submission
    properties:
      id:
        type: string
        description: The identifier of the job
      status:
        $ref: '#/definitions/JobStatus'
      submission:
        type: string
        format: date-time
        description: Submission datetime of the job in ISO8601 format
      name:
        type: string
        description: The name of the job
      start:
        type: string
        format: date-time
        description: Start datetime of the job in ISO8601 format
      end:
        type: string
        format: date-time
        description: End datetime of the job in ISO8601 format
      inputs:
        type: object
        description: Map of input keys to input values
      outputs:
        type: object
        description: Map of output keys to output values
      labels:
        type: object
        description: Custom job labels with string values
      failures:
        type: array
        items:
          $ref: '#/definitions/FailureMessage'
      extensions:
        $ref: '#/definitions/ExtendedFields'

  JobOperationResponse:
    description: Google Pipelines API provided details about an operation
    properties:
      details:
        type: string
      id:
        type: string
        description: operation ID

  UpdateJobLabelsRequest:
    description: Update labels on a job
    properties:
      labels:
        type: object
        description: |
          Labels to be merged into the existing job labels. Overwrites the label
          values for all given keys. Existing labels on the job which are not
          referenced in this request are left unmodified.

  UpdateJobLabelsResponse:
    description: Response to an update labels request
    properties:
      labels:
        type: object
        description: All job labels, after the update has been applied.

  AggregationResponse:
    description: Jobs aggregated by label and status within the specified time frame.
    properties:
      summary:
        $ref: '#/definitions/StatusCounts'
      aggregations:
        type: array
        items:
          $ref: '#/definitions/Aggregation'

  Aggregation:
    description: Aggregation of jobs by statuses for a given label, within a specified time frame.
    properties:
      name:
        type: string
      key:
        type: string
      entries:
        type: array
        items:
          $ref: '#/definitions/AggregationEntry'

  AggregationEntry:
    description: An aggregation entry for a given label value.
    properties:
      label:
        description: The label value for this aggregation entry.
        type: string
      statusCounts:
        $ref: '#/definitions/StatusCounts'

  StatusCounts:
    description: A collection of StatusCount
    properties:
      counts:
        type: array
        items:
          $ref: '#/definitions/StatusCount'

  StatusCount:
    description: A mapping of JobStatus to number of jobs with that status.
    properties:
      status:
        $ref: '#/definitions/JobStatus'
      count:
        type: integer

  TimeFrame:
    type: string
    enum: *TIMEFRAME

  QueryJobsRequest:
    description: Job query parameters
    properties:
      start:
        type: string
        format: date-time
        description: >
          Returns only jobs with an equal or later start datetime. If both start
          and end date are specified, start date must be before or equal to end
          date.
      end:
        type: string
        format: date-time
        description: >
          Returns only jobs with an equal or earlier end datetime. If both start
          and end date are specified, start date must be before or equal to end
          date.
      submission:
        type: string
        format: date-time
        description: >
          Returns only jobs with an equal or later submission datetime.
      status:
        type: array
        items:
          $ref: '#/definitions/JobStatus'
        description: >
          If specified, returns only jobs matching any of these statuses.
      labels:
        type: object
        description: >
          If specified, returns only jobs containing labels with exact string
          matches to each of these labels.
      id:
        type: string
        pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]+$
        description: >
          Returns only jobs with the specified id.
      name:
        type: string
        pattern: ^[a-zA-Z][a-zA-Z0-9_-]*$
        description: >
          Returns only jobs with the specified name.
      pageSize:
        type: integer
        format: int32
        description: |
          The maximum number of results to return in a single page. If unspecified,
          a server default will be used. Note that the server may elect to
          return fewer results per page than requested.
      pageToken:
        type: string
        description: |
          The continuation token, which is used to page through large result
          sets. To get the next page of results, set this parameter to the value
          of `nextPageToken` from the previous response.
      extensions:
        $ref: '#/definitions/ExtendedQueryFields'

  ExtendedQueryFields:
    description: >
      Extended fields to query by which may not be supported by all backends.
    properties:
      projectId:
        type: string
        description: >
          Returns only jobs belonging to the specified project. If specified by
          the /capabilities endpoint, this field is required for all query
          requests. All jobs have exactly one parent projectId.
      userId:
        type: string
        description: >
          Returns only jobs with the specified user ID.
      hideArchived:
        type: boolean
        description: >
          Returns only jobs that do not have the label 'flag' set to 'archive'

  QueryJobsResponse:
    description: Response to a job query
    required:
      - results
    properties:
      results:
        type: array
        required: true
        items:
          $ref: '#/definitions/QueryJobsResult'
      totalSize:
        type: number
        description:
          The total number of results which matched the query. Can be empty
          if the backend cannot calculate this total.
      nextPageToken:
        type: string
        description: |
          The continuation token, which is used to page through large result
          sets. Provide this value in a subsequent request to return the next
          page of results. This field will be empty if there aren't any
          additional results.

  # TODO(calbach): Reconcile QueryJobsResult with JobMetadata.
  QueryJobsResult:
    description: Result for an individual job returned by a job query
    required:
      - id
      - status
      - submission
    properties:
      id:
        type: string
        description: Job ID
      name:
        type: string
        description: Job name
      status:
        $ref: '#/definitions/JobStatus'
      submission:
        type: string
        format: date-time
        description: Submission datetime in ISO8601 format
      start:
        type: string
        format: date-time
        description: Job start datetime in ISO8601 format
      end:
        type: string
        format: date-time
        description: Job end datetime in ISO8601 format
      labels:
        type: object
        description: Custom job labels with string values
      extensions:
        $ref: '#/definitions/ExtendedFields'

  JobStatus:
    type: string
    enum:
      - Submitted
      - OnHold
      - Running
      - Aborting
      - Failed
      - Succeeded
      - Aborted

  ExtendedFields:
    description: >
      Extended fields on jobs which may not be populated by all backends. See
      the /capabilities endpoint for more details.
    properties:
      userId:
        type: string
        description: The user associated with the job.
      statusDetail:
        type: string
        description: Longer text description of the job status.
      logs:
        type: object
        description: Map of type of log file to its file location.
      lastUpdate:
        type: string
        format: date-time
        description: Job last update datetime in ISO8601 format.
      envs:
        type: object
        description: Map of ENV variables key values associated with the job.
      sourceFile:
        type: string
        description: The text of the script executed by this job.
      parentJobId:
        type: string
        description: The parent job ID for the job.
      tasks:
        type: array
        items:
          $ref: '#/definitions/TaskMetadata'
      events:
        type: array
        items:
          $ref: '#/definitions/EventDetail'

  EventDetail:
    description: Event with timestamp.
    properties:
        time:
          type: string
          format: date-time
        content:
          type: string

  TaskMetadata:
    description: Task level metadata
    properties:
      inputs:
        type: object
        description: Mapping of input fully qualified names to stringified values
      outputs:
        type: object
        description: Mapping of input fully qualified names to stringified values
      # TODO: Define a generalized enum for this. For details, see
      # https://github.com/DataBiosphere/job-manager/pull/14.
      executionStatus:
        type: string
        description: Backend-specific task status.
      start:
        type: string
        format: date-time
        description: Start datetime of the task execution in ISO8601 format with milliseconds
      end:
        type: string
        format: date-time
        description: End datetime of the task execution in ISO8601 format with milliseconds
      name:
        type: string
        description: The name of the job
      shards:
        type: array
        items:
          $ref: '#/definitions/Shard'
      failureMessages:
        type: array
        items:
          type: string
      returnCode:
        type: integer
        description: Task execution return code
      backendLog:
        type: string
        description: Path to the backend log file for this task
      callRoot:
        type: string
        description: The root directory of the task execution
      operationId:
        type: string
        description: Operation Id for Google Pipelines API
      attempts:
        type: integer
        description: Number of times the task was run
      attemptsData:
        type: array
        items:
            $ref: '#/definitions/IndividualAttempt'
      jobId:
        type: string
        description: >
          Id of the job corresponding to this task, if this task is a nested
          child job. Expresses a child:parent relationship between this job
          and the job containing this task.
      callCached:
        type: boolean
        description: A flag to indicate if task was call-cached.
        default: false
      executionEvents:
        type: array
        items:
          $ref: '#/definitions/ExecutionEvent'

  IndividualAttempt:
    description: The most granular collection of data for a job.
    properties:
      attemptNumber:
        type: integer
      executionStatus:
        type: string
        description: Backend-specific task status.
      callCached:
        type: boolean
        description: A flag to indicate if task was call-cached.
        default: false
      backendLog:
        type: string
        description: Path to the backend log file for this task
      callRoot:
        type: string
        description: The root directory of the task execution
      operationId:
        type: string
        description: Operation Id for Google Pipelines API
      inputs:
        type: object
        description: Mapping of input fully qualified names to stringified values
      outputs:
        type: object
        description: Mapping of input fully qualified names to stringified values
      start:
        type: string
        format: date-time
        description: Start datetime of the task execution in ISO8601 format with milliseconds
      end:
        type: string
        format: date-time
        description: End datetime of the task execution in ISO8601 format with milliseconds
      failureMessages:
        type: array
        items:
          type: string

  JobAttemptsResponse:
    description: Attempts for specified task or shard
    properties:
      attempts:
        type: array
        items:
          $ref: '#/definitions/IndividualAttempt'

  Shard:
    description: Individual shards for a scattered task or sub-workflow
    properties:
      executionStatus:
        type: string
        description: Backend-specific task status.
      start:
        type: string
        format: date-time
        description: Start datetime of the task execution in ISO8601 format with milliseconds
      end:
        type: string
        format: date-time
        description: End datetime of the task execution in ISO8601 format with milliseconds
      shardIndex:
        type: integer
      backendLog:
        type: string
        description: If from a task, path to the backend log file for this task or shard
      callRoot:
        type: string
        description: If from a task, the root directory of the task or shard execution
      operationId:
        type: string
        description: Operation Id for Google Pipelines API
      attempts:
        type: integer
        description: Number of times the task was run
      failureMessages:
        type: array
        items:
          type: string
      attemptsData:
        type: array
        items:
          $ref: '#/definitions/IndividualAttempt'
      jobId:
        type: string
        description: >
          Id of the job corresponding to this shard, if this is a scattered sub-workflow shard

  FailureMessage:
    description: >
      Failure messages with optional additional information if the failure
      was from a task or scattered task shard
    required:
      - failure
    properties:
      taskName:
        type: string
        description: If from a task, the name of the task that generated the failure message
      failure:
        type: string
        description: The failure message
      timestamp:
        type: string
        format: date-time
        description: The time at which this failure occurred
      backendLog:
        type: string
        description: If from a task, path to the backend log file for this task or shard
      callRoot:
        type: string
        description: If from a task, the root directory of the task or shard execution
      operationId:
        type: string
        description: Operation Id for Google Pipelines API
      shardIndex:
        type: integer
      jobId:
        type: string
        description: >
          Id of the job corresponding to this task, if this task is a nested
          child job. Expresses a child:parent relationship between this job
          and the job containing this task.

  FieldType:
    type: string
    default: text
    enum:
      - text
      - date
      - list
  BulkLabelField:
    description: a DisplayField that can be bulk-editable via the UI
    required:
      - displayField
    properties:
      default:
        type: string
      displayField:
        $ref: '#/definitions/DisplayField'

  ShardStatusCount:
    description: a total of the number of shards for a task with a certain status
    properties:
      status:
        $ref: '#/definitions/JobStatus'
      count:
        type: integer

  ExecutionEvent:
    description:
    properties:
      name:
        type: string
        description: Name of event
      start:
        type: string
        format: date-time
        description: Start datetime of the event in ISO8601 format
      end:
        type: string
        format: date-time
        description: End datetime of the event in ISO8601 format
      attemptNumber:
        type: integer
      shardIndex:
        type: integer

  HealthResponse:
    description: response from the health endpoint
    properties:
      available:
        description: Whether or not this service is available
        type: boolean

  FileContents:
    description: The contents of a file in Google Storage
    required:
      - contents
      - truncated
    properties:
      contents:
        type: string
      truncated:
        description: Whether or not the file was truncated because of size
        type: string


responses:
  BadRequest:
    description: 'Bad request'
  ServerError:
    description: 'Internal server error'
  NotFound:
    description: 'Job not found'
  JobTerminal:
    description: 'Job already in terminal status'
  Unauthorized:
    description: 'Unauthorized'
  ServiceUnavailable:
    description: 'The service or one of its requirements is unavailable'
securityDefinitions:
  google_id_token:
    type: oauth2
    description: Bearer token authorization
    name: Authorization
    in: header
    flow: implicit
    authorizationUrl: ""
    x-google-issuer: "https://accounts.google.com"
    x-google-jwks_uri: "https://www.googleapis.com/oauth2/v1/certs"
    scopes:
      https://www.googleapis.com/auth/genomics:
        Full access to google genomics and pipelines API
      https://www.googleapis.com/auth/cloudplatformprojects.readonly:
        Read-only access to Google Cloud projects
security:
  google_id_token: []