Skip to content

Commit

Permalink
Refactor spec tester to improve reusability of classes (#302)
Browse files Browse the repository at this point in the history
* Refactor to improve re-usability of classes

Signed-off-by: Thomas Farr <[email protected]>

* Extract common OpenSearch connection configuration

Signed-off-by: Thomas Farr <[email protected]>

* Update dev guide

Signed-off-by: Thomas Farr <[email protected]>

* Lift up composition of test runner

Signed-off-by: Thomas Farr <[email protected]>

* Add changelog entry

Signed-off-by: Thomas Farr <[email protected]>

* Don't verify certs in tests

Signed-off-by: Thomas Farr <[email protected]>

* Set OpenSearch password

Signed-off-by: Thomas Farr <[email protected]>

* Correct test runner test

Signed-off-by: Thomas Farr <[email protected]>

* Type safe scrub errors

Signed-off-by: Thomas Farr <[email protected]>

* Rename SpecParser

Signed-off-by: Thomas Farr <[email protected]>

---------

Signed-off-by: Thomas Farr <[email protected]>
  • Loading branch information
Xtansia committed Jun 6, 2024
1 parent 6711e8f commit 698266e
Show file tree
Hide file tree
Showing 21 changed files with 571 additions and 467 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/analyze-pr-changes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
npm install
npm run dump-cluster-spec -- --insecure --output $CLUSTER_SPEC
npm run dump-cluster-spec -- --opensearch-insecure --output $CLUSTER_SPEC
docker stop opensearch
env:
Expand Down
7 changes: 2 additions & 5 deletions .github/workflows/test-spec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ jobs:
env:
OPENSEARCH_VERSION: 2.12.0
OPENSEARCH_PASSWORD: myStrongPassword123!
OPENSEARCH_URL: https://localhost:9200
steps:
- name: Checkout Repo
uses: actions/checkout@v4
Expand All @@ -37,9 +36,7 @@ jobs:

- name: Run OpenSearch Cluster
working-directory: .github/opensearch-cluster
run: |
docker-compose up -d
sleep 60
run: docker-compose up -d && sleep 60

- name: Run Tests
run: npm run test:spec
run: npm run test:spec -- --opensearch-insecure
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
### Changed

- Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))
- Refactored spec tester internals to improve reusability ([#302](https://github.com/opensearch-project/opensearch-api-specification/pull/302))

### Deprecated

Expand Down
12 changes: 5 additions & 7 deletions DEVELOPER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -284,12 +284,10 @@ The dump-cluster-spec tool connects to an OpenSearch cluster which has the [open

#### Arguments

- `--host <host>`: The host at which the cluster is accessible, defaults to `localhost`.
- `--port <port>`: The port at which the cluster is accessible, defaults to `9200`.
- `--no-https`: Disable HTTPS, defaults to using HTTPS.
- `--insecure`: Disable SSL/TLS certificate verification, defaults to performing verification.
- `--username <username>`: The username to authenticate with the cluster, defaults to `admin`, only used when `--password` is set.
- `--password <password>`: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable.
- `--opensearch-url <url>`: The URL at which the cluster is accessible, defaults to `https://localhost:9200`.
- `--opensearch-insecure`: Disable SSL/TLS certificate verification, defaults to performing verification.
- `--opensearch-username <username>`: The username to authenticate with the cluster, defaults to `admin`, only used when `--opensearch-password` is set.
- `--opensearch-password <password>`: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable.
- `--output <path>`: The path to write the dumped spec to, defaults to `<repository-root>/build/opensearch-openapi-CLUSTER.yaml`.

#### Example
Expand All @@ -308,7 +306,7 @@ docker run \
-e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_PASSWORD" \
opensearch-with-api-plugin

OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" npm run dump-cluster-spec -- --insecure
OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" npm run dump-cluster-spec -- --opensearch-insecure

docker stop opensearch
```
Expand Down
132 changes: 132 additions & 0 deletions tools/src/OpenSearchHttpClient.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import { Option } from '@commander-js/extra-typings'
import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from 'axios'
import * as https from 'node:https'
import { sleep } from '../helpers'

const DEFAULT_URL = 'https://localhost:9200'
const DEFAULT_USER = 'admin'
const DEFAULT_INSECURE = false

export const OPENSEARCH_URL_OPTION = new Option('--opensearch-url <url>', 'URL at which the OpenSearch cluster is accessible')
.default(DEFAULT_URL)
.env('OPENSEARCH_URL')

export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username <username>', 'username to use when authenticating with OpenSearch')
.default(DEFAULT_USER)
.env('OPENSEARCH_USERNAME')

export const OPENSEARCH_PASSWORD_OPTION = new Option('--opensearch-password <password>', 'password to use when authenticating with OpenSearch')
.env('OPENSEARCH_PASSWORD')

export const OPENSEARCH_INSECURE_OPTION = new Option('--opensearch-insecure', 'disable SSL/TLS certificate verification when connecting to OpenSearch')
.default(DEFAULT_INSECURE)

export interface OpenSearchHttpClientOptions {
url?: string
username?: string
password?: string
insecure?: boolean
}

export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize<K>}`]: OpenSearchHttpClientOptions[K] }

export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptions): OpenSearchHttpClientOptions {
return {
url: opts.opensearchUrl,
username: opts.opensearchUsername,
password: opts.opensearchPassword,
insecure: opts.opensearchInsecure
}
}

export interface OpenSearchInfo {
cluster_name: string
cluster_uuid: string
name: string
tagline: string
version: {
build_date: string
build_flavor: string
build_hash: string
build_snapshot: boolean
build_type: string
lucene_version: string
minimum_index_compatibility_version: string
minimum_wire_compatibility_version: string
number: string
}
}

export class OpenSearchHttpClient {
private readonly _axios: AxiosInstance

constructor (opts?: OpenSearchHttpClientOptions) {
this._axios = axios.create({
baseURL: opts?.url ?? DEFAULT_URL,
auth: opts?.username !== undefined && opts.password !== undefined
? {
username: opts.username,
password: opts.password
}
: undefined,
httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) })
})
}

async wait_until_available (max_attempts: number = 20, wait_between_attempt_millis: number = 5000): Promise<OpenSearchInfo> {
let attempt = 0
while (true) {
attempt += 1
try {
const info = await this.get('/')
return info.data
} catch (e) {
if (attempt >= max_attempts) {
throw e
}
await sleep(wait_between_attempt_millis)
}
}
}

async request<T = any, R = AxiosResponse<T>, D = any>(config: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.request(config)
}

async get<T = any, R = AxiosResponse<T>, D = any>(url: string, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.get(url, config)
}

async delete<T = any, R = AxiosResponse<T>, D = any>(url: string, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.delete(url, config)
}

async head<T = any, R = AxiosResponse<T>, D = any>(url: string, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.head(url, config)
}

async options<T = any, R = AxiosResponse<T>, D = any>(url: string, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.options(url, config)
}

async post<T = any, R = AxiosResponse<T>, D = any>(url: string, data?: D, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.post(url, data, config)
}

async put<T = any, R = AxiosResponse<T>, D = any>(url: string, data?: D, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.put(url, data, config)
}

async patch<T = any, R = AxiosResponse<T>, D = any>(url: string, data?: D, config?: AxiosRequestConfig<D>): Promise<R> {
return await this._axios.patch(url, data, config)
}
}
64 changes: 21 additions & 43 deletions tools/src/dump-cluster-spec/dump-cluster-spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,68 +9,46 @@

import { Command, Option } from '@commander-js/extra-typings'
import { resolve } from 'path'
import axios from 'axios'
import * as https from 'node:https'
import * as process from 'node:process'
import { sleep, write_yaml } from '../../helpers'
import { write_yaml } from '../../helpers'
import {
get_opensearch_opts_from_cli,
OPENSEARCH_INSECURE_OPTION,
OPENSEARCH_PASSWORD_OPTION,
OPENSEARCH_URL_OPTION,
OPENSEARCH_USERNAME_OPTION, OpenSearchHttpClient,
type OpenSearchHttpClientOptions
} from '../OpenSearchHttpClient'

interface CommandOpts {
host: string
https: boolean
insecure: boolean
port: string | number
username: string
password?: string
opensearch: OpenSearchHttpClientOptions
output: string
}

async function main (opts: CommandOpts): Promise<void> {
const url = `http${opts.https ? 's' : ''}://${opts.host}:${opts.port}`
const client = axios.create({
httpsAgent: new https.Agent({
rejectUnauthorized: !opts.insecure
}),
auth: opts.password !== undefined
? {
username: opts.username,
password: opts.password
}
: undefined
})
const client = new OpenSearchHttpClient(opts.opensearch)

let attempt = 0
while (true) {
attempt += 1
try {
const info = await client.get(url)
console.log(info.data)
break
} catch (e) {
if (attempt >= 20) {
throw e
}
await sleep(5000)
}
}
const info = await client.wait_until_available()
console.log(info)

const cluster_spec = await client.get(`${url}/_plugins/api`)
const cluster_spec = await client.get('/_plugins/api')

write_yaml(opts.output, cluster_spec.data)
}

const command = new Command()
.description('Dumps an OpenSearch cluster\'s generated specification.')
.addOption(new Option('--host <host>', 'cluster\'s host').default('localhost'))
.addOption(new Option('--no-https', 'disable HTTPS'))
.addOption(new Option('--insecure', 'disable SSL/TLS certificate verification').default(false))
.addOption(new Option('--port <port>', 'cluster\'s port to connect to').default(9200))
.addOption(new Option('--username <username>', 'username to authenticate with the cluster').default('admin'))
.addOption(new Option('--password <password>', 'password to authenticate with the cluster').env('OPENSEARCH_PASSWORD'))
.addOption(OPENSEARCH_URL_OPTION)
.addOption(OPENSEARCH_USERNAME_OPTION)
.addOption(OPENSEARCH_PASSWORD_OPTION)
.addOption(OPENSEARCH_INSECURE_OPTION)
.addOption(new Option('--output <path>', 'path to the output file').default(resolve(__dirname, '../../../build/opensearch-openapi-CLUSTER.yaml')))
.allowExcessArguments(false)
.parse()

main(command.opts())
const opts = command.opts()

main({ output: opts.output, opensearch: get_opensearch_opts_from_cli(opts) })
.catch(e => {
if (e instanceof Error) {
console.error(`ERROR: ${e.stack}`)
Expand Down
64 changes: 29 additions & 35 deletions tools/src/tester/ChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,76 +12,70 @@ import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.ty
import { type ParsedOperation } from './types/spec.types'
import { overall_result } from './helpers'
import type ChapterReader from './ChapterReader'
import SharedResources from './SharedResources'
import type SpecParser from './SpecParser'
import type OperationLocator from './OperationLocator'
import type SchemaValidator from './SchemaValidator'

export default class ChapterEvaluator {
chapter: Chapter
skip_payload_evaluation: boolean = false
spec_parser: SpecParser
chapter_reader: ChapterReader
schema_validator: SchemaValidator
private readonly _operation_locator: OperationLocator
private readonly _chapter_reader: ChapterReader
private readonly _schema_validator: SchemaValidator

constructor (chapter: Chapter) {
this.chapter = chapter
this.spec_parser = SharedResources.get_instance().spec_parser
this.chapter_reader = SharedResources.get_instance().chapter_reader
this.schema_validator = SharedResources.get_instance().schema_validator
constructor (spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator) {
this._operation_locator = spec_parser
this._chapter_reader = chapter_reader
this._schema_validator = schema_validator
}

async evaluate (skip: boolean): Promise<ChapterEvaluation> {
if (skip) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
const response = await this.chapter_reader.read(this.chapter)
const operation = this.spec_parser.locate_operation(this.chapter)
if (operation == null) return { title: this.chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${this.chapter.method.toUpperCase()} ${this.chapter.path}" not found in the spec.` } }
const params = this.#evaluate_parameters(operation)
const request_body = this.#evaluate_request_body(operation)
const status = this.#evaluate_status(response)
const payload = this.#evaluate_payload(operation, response)
async evaluate (chapter: Chapter, skip: boolean): Promise<ChapterEvaluation> {
if (skip) return { title: chapter.synopsis, overall: { result: Result.SKIPPED } }
const response = await this._chapter_reader.read(chapter)
const operation = this._operation_locator.locate_operation(chapter)
if (operation == null) return { title: chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${chapter.method.toUpperCase()} ${chapter.path}" not found in the spec.` } }
const params = this.#evaluate_parameters(chapter, operation)
const request_body = this.#evaluate_request_body(chapter, operation)
const status = this.#evaluate_status(chapter, response)
const payload = status.result === Result.PASSED ? this.#evaluate_payload(response, operation) : { result: Result.SKIPPED }
return {
title: this.chapter.synopsis,
title: chapter.synopsis,
overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
request: { parameters: params, request_body },
response: { status, payload }
}
}

#evaluate_parameters (operation: ParsedOperation): Record<string, Evaluation> {
return Object.fromEntries(Object.entries(this.chapter.parameters ?? {}).map(([name, parameter]) => {
#evaluate_parameters (chapter: Chapter, operation: ParsedOperation): Record<string, Evaluation> {
return Object.fromEntries(Object.entries(chapter.parameters ?? {}).map(([name, parameter]) => {
const schema = operation.parameters[name]?.schema
if (schema == null) return [name, { result: Result.FAILED, message: `Schema for "${name}" parameter not found.` }]
const evaluation = this.schema_validator.validate(schema, parameter)
const evaluation = this._schema_validator.validate(schema, parameter)
return [name, evaluation]
}))
}

#evaluate_request_body (operation: ParsedOperation): Evaluation {
if (!this.chapter.request_body) return { result: Result.PASSED }
const content_type = this.chapter.request_body.content_type ?? 'application/json'
#evaluate_request_body (chapter: Chapter, operation: ParsedOperation): Evaluation {
if (!chapter.request_body) return { result: Result.PASSED }
const content_type = chapter.request_body.content_type ?? 'application/json'
const schema = operation.requestBody?.content[content_type]?.schema
if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` }
return this.schema_validator.validate(schema, this.chapter.request_body?.payload ?? {})
return this._schema_validator.validate(schema, chapter.request_body?.payload ?? {})
}

#evaluate_status (response: ActualResponse): Evaluation {
const expected_status = this.chapter.response?.status ?? 200
#evaluate_status (chapter: Chapter, response: ActualResponse): Evaluation {
const expected_status = chapter.response?.status ?? 200
if (response.status === expected_status) return { result: Result.PASSED }
this.skip_payload_evaluation = true
return {
result: Result.ERROR,
message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}. ${response.message}`,
error: response.error as Error
}
}

#evaluate_payload (operation: ParsedOperation, response: ActualResponse): Evaluation {
if (this.skip_payload_evaluation) return { result: Result.SKIPPED }
#evaluate_payload (response: ActualResponse, operation: ParsedOperation): Evaluation {
const content_type = response.content_type ?? 'application/json'
const content = operation.responses[response.status]?.content[content_type]
const schema = content?.schema
if (schema == null && content != null) return { result: Result.PASSED }
if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` }
return this.schema_validator.validate(schema, response.payload)
return this._schema_validator.validate(schema, response.payload)
}
}
Loading

0 comments on commit 698266e

Please sign in to comment.