From 597ed8d73c0b72743ed90fd9694ad9ba511d5be7 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Mon, 12 Aug 2024 22:59:54 +1200 Subject: [PATCH] Improve indices.stats schemas (#491) * Improve indices.stats schemas Signed-off-by: Thomas Farr * Add changelog Signed-off-by: Thomas Farr * Remove unused schemas Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/namespaces/indices.yaml | 8 +- spec/schemas/_common.yaml | 147 +++++++++--------------- spec/schemas/indices.stats.yaml | 195 +++++++++----------------------- spec/schemas/nodes._common.yaml | 2 +- 5 files changed, 118 insertions(+), 235 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a941ea03..a5821316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430)) - Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) +- Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) ### Security diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index 6d36982f..55f0c299 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2686,14 +2686,14 @@ components: schema: type: object properties: + _shards: + $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' + _all: + $ref: '../schemas/indices.stats.yaml#/components/schemas/AllIndicesStats' indices: type: object additionalProperties: $ref: '../schemas/indices.stats.yaml#/components/schemas/IndicesStats' - _shards: - $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' - _all: - $ref: '../schemas/indices.stats.yaml#/components/schemas/IndicesStats' required: - _all - _shards diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 5b645eed..f89bdb9b 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1250,13 +1250,6 @@ components: reserved_in_bytes: description: A prediction, in bytes, of how much larger the shard stores will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. type: number - total_data_set_size: - $ref: '#/components/schemas/ByteSize' - total_data_set_size_in_bytes: - description: |- - Total data set size, in bytes, of all shards assigned to selected nodes. - This includes the size of shards not stored fully on the nodes, such as the cache for partially mounted indices. - type: number required: - reserved_in_bytes - size_in_bytes @@ -1503,25 +1496,25 @@ components: GetStats: type: object properties: - current: + total: type: number - exists_time: + time: $ref: '#/components/schemas/Duration' - exists_time_in_millis: + time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' exists_total: type: number - missing_time: + exists_time: $ref: '#/components/schemas/Duration' - missing_time_in_millis: + exists_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' missing_total: type: number - time: + missing_time: $ref: '#/components/schemas/Duration' - time_in_millis: + missing_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - total: + current: type: number required: - current @@ -1534,40 +1527,39 @@ components: IndexingStats: type: object properties: + index_total: + type: number + index_time: + $ref: '#/components/schemas/Duration' + index_time_in_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' index_current: type: number - delete_current: + index_failed: + type: number + delete_total: type: number delete_time: $ref: '#/components/schemas/Duration' delete_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - delete_total: + delete_current: type: number - doc_status: - $ref: '#/components/schemas/DocStatus' - is_throttled: - type: boolean noop_update_total: type: number + is_throttled: + type: boolean throttle_time: $ref: '#/components/schemas/Duration' throttle_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - index_time: - $ref: '#/components/schemas/Duration' - index_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - index_total: - type: number - index_failed: - type: number + doc_status: + $ref: '#/components/schemas/DocStatus' types: + x-version-removed: '2.0' type: object additionalProperties: $ref: '#/components/schemas/IndexingStats' - write_load: - type: number required: - delete_current - delete_time_in_millis @@ -1684,6 +1676,21 @@ components: SearchStats: type: object properties: + open_contexts: + description: The number of open search contexts. + type: number + query_current: + description: The number of shard query operations that are currently running. + type: number + query_time: + description: The total amount of time for all shard query operations. + $ref: '#/components/schemas/Duration' + query_time_in_millis: + description: The total amount of time for all shard query operations, in milliseconds. + $ref: '#/components/schemas/DurationValueUnitMillis' + query_total: + description: The total number of shard query operations. + type: number concurrent_query_total: description: The total number of query operations that use concurrent segment search. type: number @@ -1710,39 +1717,6 @@ components: fetch_total: description: The total number of shard fetch operations. type: number - open_contexts: - description: The number of open search contexts. - type: number - point_in_time_total: - description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. - type: number - point_in_time_time: - $ref: '#/components/schemas/Duration' - point_in_time_time_in_millis: - description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. - $ref: '#/components/schemas/DurationValueUnitMillis' - point_in_time_current: - description: The number of shard PIT contexts currently open. - type: number - query_current: - description: The number of shard query operations that are currently running. - type: number - query_time: - description: The total amount of time for all shard query operations. - $ref: '#/components/schemas/Duration' - query_time_in_millis: - description: The total amount of time for all shard query operations, in milliseconds. - $ref: '#/components/schemas/DurationValueUnitMillis' - query_total: - description: The total number of shard query operations. - type: number - request: - type: object - description: Statistics about coordinator search operations for the node. - additionalProperties: - $ref: '#/components/schemas/RequestStats' - search_idle_reactivate_count_total: - type: number scroll_current: description: The number of shard scroll operations that are currently running. type: number @@ -1755,6 +1729,17 @@ components: scroll_total: description: The total number of shard scroll operations. type: number + point_in_time_total: + description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. + type: number + point_in_time_time: + $ref: '#/components/schemas/Duration' + point_in_time_time_in_millis: + description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. + $ref: '#/components/schemas/DurationValueUnitMillis' + point_in_time_current: + description: The number of shard PIT contexts currently open. + type: number suggest_current: description: The number of shard suggest operations that are currently running. type: number @@ -1767,6 +1752,13 @@ components: suggest_total: description: The total number of shard suggest operations. type: number + search_idle_reactivate_count_total: + type: number + request: + type: object + description: Statistics about coordinator search operations for the node. + additionalProperties: + $ref: '#/components/schemas/RequestStats' groups: type: object additionalProperties: @@ -1833,33 +1825,6 @@ components: - current - total - total_time_in_millis - BulkStats: - type: object - properties: - total_operations: - type: number - total_time: - $ref: '#/components/schemas/Duration' - total_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - total_size: - $ref: '#/components/schemas/ByteSize' - total_size_in_bytes: - type: number - avg_time: - $ref: '#/components/schemas/Duration' - avg_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - avg_size: - $ref: '#/components/schemas/ByteSize' - avg_size_in_bytes: - type: number - required: - - avg_size_in_bytes - - avg_time_in_millis - - total_operations - - total_size_in_bytes - - total_time_in_millis GeoShapeRelation: type: string enum: diff --git a/spec/schemas/indices.stats.yaml b/spec/schemas/indices.stats.yaml index d3301fcd..eb729afb 100644 --- a/spec/schemas/indices.stats.yaml +++ b/spec/schemas/indices.stats.yaml @@ -24,130 +24,89 @@ components: required: - description - size_in_bytes + AllIndicesStats: + type: object + properties: + primaries: + $ref: '#/components/schemas/IndexStats' + total: + $ref: '#/components/schemas/IndexStats' + required: + - primaries + - total IndicesStats: type: object properties: + uuid: + $ref: '_common.yaml#/components/schemas/Uuid' primaries: $ref: '#/components/schemas/IndexStats' + total: + $ref: '#/components/schemas/IndexStats' shards: type: object additionalProperties: type: array items: - $ref: '#/components/schemas/ShardStats' - total: - $ref: '#/components/schemas/IndexStats' - uuid: - $ref: '_common.yaml#/components/schemas/Uuid' - health: - $ref: '_common.yaml#/components/schemas/HealthStatus' - status: - $ref: '#/components/schemas/IndexMetadataState' - IndexStats: + $ref: '#/components/schemas/IndexShardStats' + required: + - primaries + - total + - uuid + IndexStatsBase: type: object properties: - completion: - $ref: '_common.yaml#/components/schemas/CompletionStats' docs: $ref: '_common.yaml#/components/schemas/DocStats' - fielddata: - $ref: '_common.yaml#/components/schemas/FielddataStats' - flush: - $ref: '_common.yaml#/components/schemas/FlushStats' - get: - $ref: '_common.yaml#/components/schemas/GetStats' + store: + $ref: '_common.yaml#/components/schemas/StoreStats' indexing: $ref: '_common.yaml#/components/schemas/IndexingStats' - indices: - $ref: '#/components/schemas/IndicesStats' + get: + $ref: '_common.yaml#/components/schemas/GetStats' + search: + $ref: '_common.yaml#/components/schemas/SearchStats' merges: $ref: '_common.yaml#/components/schemas/MergesStats' - query_cache: - $ref: '_common.yaml#/components/schemas/QueryCacheStats' - recovery: - $ref: '_common.yaml#/components/schemas/RecoveryStats' refresh: $ref: '_common.yaml#/components/schemas/RefreshStats' - request_cache: - $ref: '_common.yaml#/components/schemas/RequestCacheStats' - search: - $ref: '_common.yaml#/components/schemas/SearchStats' - segments: - $ref: '_common.yaml#/components/schemas/SegmentsStats' - store: - $ref: '_common.yaml#/components/schemas/StoreStats' - translog: - $ref: '_common.yaml#/components/schemas/TranslogStats' + flush: + $ref: '_common.yaml#/components/schemas/FlushStats' warmer: $ref: '_common.yaml#/components/schemas/WarmerStats' - bulk: - $ref: '_common.yaml#/components/schemas/BulkStats' - shard_stats: - $ref: '#/components/schemas/ShardsTotalStats' - ShardsTotalStats: - type: object - properties: - total_count: - type: number - required: - - total_count - ShardStats: - type: object - properties: - commit: - $ref: '#/components/schemas/ShardCommit' - completion: - $ref: '_common.yaml#/components/schemas/CompletionStats' - docs: - $ref: '_common.yaml#/components/schemas/DocStats' + query_cache: + $ref: '_common.yaml#/components/schemas/QueryCacheStats' fielddata: $ref: '_common.yaml#/components/schemas/FielddataStats' - flush: - $ref: '_common.yaml#/components/schemas/FlushStats' - get: - $ref: '_common.yaml#/components/schemas/GetStats' - indexing: - $ref: '_common.yaml#/components/schemas/IndexingStats' - mappings: - $ref: '#/components/schemas/MappingStats' - merges: - $ref: '_common.yaml#/components/schemas/MergesStats' - shard_path: - $ref: '#/components/schemas/ShardPath' - query_cache: - $ref: '#/components/schemas/ShardQueryCache' - recovery: - $ref: '_common.yaml#/components/schemas/RecoveryStats' - refresh: - $ref: '_common.yaml#/components/schemas/RefreshStats' - request_cache: - $ref: '_common.yaml#/components/schemas/RequestCacheStats' - retention_leases: - $ref: '#/components/schemas/ShardRetentionLeases' - routing: - $ref: '#/components/schemas/ShardRouting' - search: - $ref: '_common.yaml#/components/schemas/SearchStats' + completion: + $ref: '_common.yaml#/components/schemas/CompletionStats' segments: $ref: '_common.yaml#/components/schemas/SegmentsStats' - seq_no: - $ref: '#/components/schemas/ShardSequenceNumber' - store: - $ref: '_common.yaml#/components/schemas/StoreStats' translog: $ref: '_common.yaml#/components/schemas/TranslogStats' - warmer: - $ref: '_common.yaml#/components/schemas/WarmerStats' - bulk: - $ref: '_common.yaml#/components/schemas/BulkStats' - shards: - type: object - additionalProperties: - type: object - shard_stats: - $ref: '#/components/schemas/ShardsTotalStats' - indices: - $ref: '#/components/schemas/IndicesStats' + request_cache: + $ref: '_common.yaml#/components/schemas/RequestCacheStats' + recovery: + $ref: '_common.yaml#/components/schemas/RecoveryStats' + IndexStats: + allOf: + - $ref: '#/components/schemas/IndexStatsBase' + - type: object + IndexShardStats: + allOf: + - $ref: '#/components/schemas/IndexStatsBase' + - type: object + properties: + routing: + $ref: '#/components/schemas/ShardRouting' + commit: + $ref: '#/components/schemas/ShardCommit' + seq_no: + $ref: '#/components/schemas/ShardSequenceNumber' + retention_leases: + $ref: '#/components/schemas/ShardRetentionLeases' + shard_path: + $ref: '#/components/schemas/ShardPath' ShardCommit: type: object properties: @@ -166,18 +125,6 @@ components: - id - num_docs - user_data - MappingStats: - type: object - properties: - total_count: - type: number - total_estimated_overhead: - $ref: '_common.yaml#/components/schemas/ByteSize' - total_estimated_overhead_in_bytes: - type: number - required: - - total_count - - total_estimated_overhead_in_bytes ShardPath: type: object properties: @@ -191,31 +138,6 @@ components: - data_path - is_custom_data_path - state_path - ShardQueryCache: - type: object - properties: - cache_count: - type: number - cache_size: - type: number - evictions: - type: number - hit_count: - type: number - memory_size_in_bytes: - type: number - miss_count: - type: number - total_count: - type: number - required: - - cache_count - - cache_size - - evictions - - hit_count - - memory_size_in_bytes - - miss_count - - total_count ShardRetentionLeases: type: object properties: @@ -281,9 +203,4 @@ components: required: - global_checkpoint - local_checkpoint - - max_seq_no - IndexMetadataState: - type: string - enum: - - close - - open + - max_seq_no \ No newline at end of file diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index 59378e88..ff0efdc2 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -250,7 +250,7 @@ components: indexing_pressure: $ref: '#/components/schemas/IndexingPressure' indices: - $ref: 'indices.stats.yaml#/components/schemas/ShardStats' + $ref: 'indices.stats.yaml#/components/schemas/IndexShardStats' shard_indexing_pressure: $ref: '#/components/schemas/ShardIndexingPressureStats' search_backpressure: