diff --git a/src/Knet.Kudu.Client/Protos/kudu/client/client.proto b/src/Knet.Kudu.Client/Protos/kudu/client/client.proto index b96f42e..bc80b83 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/client/client.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/client/client.proto @@ -212,4 +212,7 @@ message AuthenticationCredentialsPB { // Trusted root CA certificates. repeated bytes ca_cert_ders = 2; + + // A JWT to be verified by the master. + optional security.JwtRawPB jwt = 4; } diff --git a/src/Knet.Kudu.Client/Protos/kudu/common/common.proto b/src/Knet.Kudu.Client/Protos/kudu/common/common.proto index ec87375..6de2221 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/common/common.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/common/common.proto @@ -138,6 +138,11 @@ message ColumnSchemaPB { // The comment for the column. optional string comment = 12; + + optional bool immutable = 13 [default = false]; + + // Whether the column is auto-incrementing. + optional bool is_auto_incrementing = 14 [default = false]; } message ColumnSchemaDeltaPB { @@ -152,6 +157,7 @@ message ColumnSchemaDeltaPB { optional int32 block_size = 8; optional string new_comment = 9; + optional bool immutable = 10 [default = false]; } message SchemaPB { @@ -310,6 +316,8 @@ enum ReplicaSelection { // - Replicas whose tablet server has the same location as the client // - All other replicas CLOSEST_REPLICA = 2; + // Select the first replica in the list. + FIRST_REPLICA = 3; } // The serialized format of a Kudu table partition schema. diff --git a/src/Knet.Kudu.Client/Protos/kudu/common/row_operations.proto b/src/Knet.Kudu.Client/Protos/kudu/common/row_operations.proto index ecca09c..a36e8b6 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/common/row_operations.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/common/row_operations.proto @@ -41,6 +41,7 @@ message RowOperationsPB { INSERT_IGNORE = 10; UPDATE_IGNORE = 11; DELETE_IGNORE = 12; + UPSERT_IGNORE = 13; // Used when specifying split rows on table creation. SPLIT_ROW = 4; diff --git a/src/Knet.Kudu.Client/Protos/kudu/common/wire_protocol.proto b/src/Knet.Kudu.Client/Protos/kudu/common/wire_protocol.proto index 95b8115..1efbe73 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/common/wire_protocol.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/common/wire_protocol.proto @@ -58,6 +58,7 @@ message AppStatusPB { INCOMPLETE = 17; END_OF_FILE = 18; CANCELLED = 19; + IMMUTABLE = 20; } required ErrorCode code = 1; @@ -87,6 +88,21 @@ message NodeInstancePB { message ServerRegistrationPB { repeated HostPortPB rpc_addresses = 1; repeated HostPortPB http_addresses = 2; + + // Addresses of this server's RPC endpoints advertised at a TCP proxy. + // It's assumed the proxy forwards RPC requests from the specified addresses + // to a dedicated RPC endpoint so that the server knows that requests came + // from the outside, and it can process it accordingly (e.g., transforming + // the internal addresses to be reachable from the outside via the proxied + // endpoints, etc.). + repeated HostPortPB rpc_proxy_addresses = 7; + + // Addresses of this server's embedded web server HTTP/HTTPS endpoints + // advertised at a TCP (HTTP?) proxy. It's assumed the proxy forwards HTTP + // requests from the specified addresses to the addresses specified in the + // 'http_addresses' field. + repeated HostPortPB http_proxy_addresses = 8; + optional string software_version = 3; // True if HTTPS has been enabled for the web interface. diff --git a/src/Knet.Kudu.Client/Protos/kudu/fs/fs.proto b/src/Knet.Kudu.Client/Protos/kudu/fs/fs.proto index a8934c4..de5666e 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/fs/fs.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/fs/fs.proto @@ -36,6 +36,15 @@ message InstanceMetadataPB { // initialized. required string format_stamp = 2; + // Encrypted server key used to encrypt/decrypt file keys on this server. + optional string server_key = 3; + + // Initialization vector for the server key. + optional string server_key_iv = 4; + + // Server key version. + optional string server_key_version = 5; + // TODO: add a "node type" (TS/Master?) } diff --git a/src/Knet.Kudu.Client/Protos/kudu/master/master.proto b/src/Knet.Kudu.Client/Protos/kudu/master/master.proto index 1f4b617..52656bd 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/master/master.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/master/master.proto @@ -90,6 +90,9 @@ message MasterErrorPB { // Master is already part of the Raft configuration. MASTER_ALREADY_PRESENT = 15; + + // The requested table is in soft_deleted state. + TABLE_SOFT_DELETED = 16; } // The error code. @@ -130,6 +133,7 @@ message SysTabletsEntryPB { RUNNING = 2; REPLACED = 3; DELETED = 4; + SOFT_DELETED = 5; } // DEPRECATED. Replaced by 'partition'. @@ -152,6 +156,9 @@ message SysTabletsEntryPB { // The dimension label for the tablet. Used for dimension-specific // placement of the tablet's replicas. optional string dimension_label = 8; + + // The delete time of the tablet, in seconds since the epoch. + optional int64 delete_timestamp = 9; } // The on-disk entry in the sys.catalog table ("metadata" column) for @@ -163,6 +170,7 @@ message SysTablesEntryPB { RUNNING = 2; ALTERING = 3; REMOVED = 4; + SOFT_DELETED = 5; } // Table name @@ -216,6 +224,12 @@ message SysTablesEntryPB { // The comment on the table. optional string comment = 17; + + // The delete time of the table, in seconds since the epoch. + optional int64 delete_timestamp = 18; + + // The reservation time interval (in seconds) between soft delete and delete. + optional uint32 soft_deleted_reserved_seconds = 19; } // The on-disk entry in the sys.catalog table ("metadata" column) to represent @@ -571,6 +585,14 @@ message DeleteTableRequestPB { // Whether to apply the deletion to external catalogs, such as the Hive Metastore, // which the Kudu master has been configured to integrate with. optional bool modify_external_catalogs = 2 [default = true]; + + // Reserve seconds after the table has been deleted. + // If this field is specified by the client, means the request coming from a newer Kudu + // client with the precise value for 'reserve_seconds', and the field's value from the request + // should be taken as-is regardless of the current setting of the '--default_deleted_table_reserve_seconds' + // flag at the server side. + // Otherwise, the behavior of DeleteRPC is controlled by the '--default_deleted_table_reserve_seconds' flag. + optional uint32 reserve_seconds = 3; } message DeleteTableResponsePB { @@ -578,6 +600,19 @@ message DeleteTableResponsePB { optional MasterErrorPB error = 1; } +message RecallDeletedTableRequestPB { + required TableIdentifierPB table = 1; + + // If this field is set, that's the name for the recalled table. + // Otherwise, the recalled table will use the original table name. + optional string new_table_name = 2; +} + +message RecallDeletedTableResponsePB { + // The error, if an error occurred with this request. + optional MasterErrorPB error = 1; +} + message ListTablesRequestPB { // When used, only returns tables that satisfy a substring match on name_filter. optional string name_filter = 1; @@ -586,18 +621,33 @@ message ListTablesRequestPB { // interpreted as if it were set to [TableTypePB::DEFAULT_TABLE], meaning // to include only user-defined tables. repeated TableTypePB type_filter = 2; + + // Set this field 'true' to include information on the partition backed by + // each tablet in the result list. + optional bool list_tablet_with_partition = 3 [default = false]; + + // Use to select the tables type for display. + // Only show regular tables if false. + // Only show soft_deleted tables if true. + optional bool show_soft_deleted = 4; } message ListTablesResponsePB { // The error, if an error occurred with this request. optional MasterErrorPB error = 1; + message TabletWithPartition { + optional string tablet_id = 1; + optional PartitionPB partition = 2; + } + message TableInfo { required bytes id = 1; required string name = 2; optional uint64 live_row_count = 3; optional int32 num_tablets = 4; optional int32 num_replicas = 5; + repeated TabletWithPartition tablet_with_partition = 6; } repeated TableInfo tables = 2; @@ -687,7 +737,7 @@ message GetTableLocationsResponsePB { // If the client caches table locations, the entries should not live longer // than this timeout. Defaults to one hour. - optional uint32 ttl_millis = 3 [default = 36000000]; + optional uint32 ttl_millis = 3 [default = 3600000]; } message AlterTableRequestPB { @@ -720,6 +770,15 @@ message AlterTableRequestPB { optional ColumnSchemaDeltaPB delta = 1; } message AddRangePartition { + // A structure to define range-specific hash schema. This separate type + // exists to distinguish from an empty hash schema (i.e. no hash bucketing) + // and the absence of range-specific hash schema when a range partition + // uses the table-wide hash schema instead. Otherwise, using a field of + // repeated HashBucketSchemaPB wouldn't allow to tell between those cases. + message CustomHashSchema { + repeated PartitionSchemaPB.HashBucketSchemaPB hash_schema = 1; + } + // A set of row operations containing the lower and upper range bound for // the range partition to add or drop. optional RowOperationsPB range_bounds = 1; @@ -727,6 +786,10 @@ message AlterTableRequestPB { // The dimension label for the tablet. Used for dimension-specific placement // of the tablet's replicas. optional string dimension_label = 2; + + // The custom hash partition schema for the range, if specified. If absent, + // the range uses table-wide hash schema. + optional CustomHashSchema custom_hash_schema = 3; } message DropRangePartition { // A set of row operations containing the lower and upper range bound for @@ -1108,6 +1171,14 @@ enum MasterFeatures { // Though this is technically a tserver feature, it's unreasonable to check if every // tablet server supports this feature. Instead we use the master as a proxy. IGNORE_OPERATIONS = 7; + // Whether master supports tables with range-specific hash schemas. + RANGE_SPECIFIC_HASH_SCHEMA = 8; + // Similar to IGNORE_OPERATIONS, but this is for UPSERT_IGNORE specifically. + UPSERT_IGNORE = 9; + // Whether master supports immutable attribute on column schema. + IMMUTABLE_COLUMN_ATTRIBUTE = 10; + // Whether master supports auto incrementing column. + AUTO_INCREMENTING_COLUMN = 11; } service MasterService { @@ -1144,6 +1215,10 @@ service MasterService { option (kudu.rpc.authz_method) = "AuthorizeClient"; } + rpc RecallDeletedTable(RecallDeletedTableRequestPB) returns (RecallDeletedTableResponsePB) { + option (kudu.rpc.authz_method) = "AuthorizeClient"; + } + rpc AlterTable(AlterTableRequestPB) returns (AlterTableResponsePB) { option (kudu.rpc.authz_method) = "AuthorizeClientOrServiceUser"; } diff --git a/src/Knet.Kudu.Client/Protos/kudu/rpc/rpc_header.proto b/src/Knet.Kudu.Client/Protos/kudu/rpc/rpc_header.proto index 56a7971..5c04877 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/rpc/rpc_header.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/rpc/rpc_header.proto @@ -101,6 +101,7 @@ message AuthenticationTypePB { message Sasl {}; message Token {}; message Certificate {}; + message Jwt {}; oneof type { // The server and client mutually authenticate via SASL. @@ -117,6 +118,11 @@ message AuthenticationTypePB { // // Certificate authentication requires the connection to be TLS encrypted. Certificate certificate = 3; + + // The server authenticates the client via a JSON web token. + // + // Requires the connection to be TLS encrypted. + Jwt jwt = 4; } } @@ -131,6 +137,7 @@ message NegotiatePB { SASL_RESPONSE = 4; TLS_HANDSHAKE = 5; TOKEN_EXCHANGE = 6; + JWT_EXCHANGE = 7; } message SaslMechanism { @@ -186,6 +193,10 @@ message NegotiatePB { // During the TOKEN_EXCHANGE step, contains the client's signed authentication token. optional security.SignedTokenPB authn_token = 8; + + // During the JWT_EXCHANGE step, contains the client's JWT used for + // authentication. + optional security.JwtRawPB jwt_raw = 10; } message RemoteMethodPB { @@ -337,6 +348,9 @@ message ErrorStatusPB { // negotiation failed, and the client should obtain a new authn token and // try to reconnect. FATAL_INVALID_AUTHENTICATION_TOKEN = 16; + + // The JWT is invalid or expired. + FATAL_INVALID_JWT = 18; } required string message = 1; diff --git a/src/Knet.Kudu.Client/Protos/kudu/security/token.proto b/src/Knet.Kudu.Client/Protos/kudu/security/token.proto index 202dbd9..fec927a 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/security/token.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/security/token.proto @@ -91,6 +91,11 @@ message TokenPB { } }; +// JSON Web Token: a wrapper to pass around a JWT as is. +message JwtRawPB { + optional bytes jwt_data = 1; +} + message SignedTokenPB { // The actual token data. This is a serialized TokenPB protobuf. However, we use a // 'bytes' field, since protobuf doesn't guarantee that if two implementations serialize diff --git a/src/Knet.Kudu.Client/Protos/kudu/tools/tool.proto b/src/Knet.Kudu.Client/Protos/kudu/tools/tool.proto index 291e49d..6ab39dc 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/tools/tool.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/tools/tool.proto @@ -69,6 +69,21 @@ message CreateClusterRequestPB { optional string renew_lifetime = 2; } optional MiniKdcOptionsPB mini_kdc_options = 8; + + // Options pertaining to a single JWKS. + message JwksOptionsPB { + optional string account_id = 1; + optional bool is_valid_key = 2 [ default = true ]; + } + + message MiniOidcOptionsPB { + // The default expiration time for JWTs. + optional string expiration_time = 1; + + // Options for JWKS to host. + repeated JwksOptionsPB jwks_options = 2; + } + optional MiniOidcOptionsPB mini_oidc_options = 11; } // Destroys a cluster created via 'create_cluster'. @@ -91,6 +106,7 @@ enum DaemonType { MASTER = 1; TSERVER = 2; KDC = 3; + JWKS = 4; } // Identifier for a cluster daemon, unique to the cluster. @@ -138,6 +154,9 @@ message DaemonInfoPB { // Daemon's bound RPC address. optional HostPortPB bound_rpc_address = 2; + + // Bound address of the embedded web server. + optional HostPortPB bound_http_address = 3; } // Response to a GetMastersRequestPB. @@ -179,6 +198,14 @@ message KinitRequestPB { optional string username = 1 [ default = "test-admin" ]; }; +message GetJwtServerResponsePB { + // Server info. + optional DaemonInfoPB servers = 1; +} + +// Gets information on each started tablet server. +message GetJwtServerRequestPB {} + // Call SetFlag() on the specific daemon. message SetDaemonFlagRequestPB { // The identifier of the daemon to sent the request to. @@ -200,6 +227,7 @@ message ControlShellResponsePB { GetMastersResponsePB get_masters = 2; GetTServersResponsePB get_tservers = 3; GetKDCEnvVarsResponsePB get_kdc_env_vars = 4; + CreateJwtResponsePB create_jwt = 5; } } @@ -226,6 +254,7 @@ message ControlShellRequestPB { SetDaemonFlagRequestPB set_daemon_flag = 12; PauseDaemonRequestPB pause_daemon = 13; ResumeDaemonRequestPB resume_daemon = 14; + CreateJwtRequestPB create_jwt = 15; } } @@ -298,6 +327,7 @@ message TabletSummaryPB { optional string status = 5; optional ConsensusStatePB master_cstate = 6; repeated ReplicaSummaryPB replicas = 7; + optional string range_key_begin = 8; } message ReplicaSummaryPB { @@ -398,6 +428,20 @@ message ColumnPB { } message PartitionPB { + message HashPartitionPB { + // Column names of columns included in the hash. Every column must be + // a component of the primary key. + repeated string columns = 1; + // Number of buckets into which columns will be hashed. Must be at least 2. + optional int32 num_buckets = 2; + // Seed value for hash calculation. Administrators may set a seed value + // on a per-table basis in order to randomize the mapping of rows to + // buckets. Setting a seed provides some amount of protection against denial + // of service attacks when the hash bucket columns contain user provided + // input. + optional uint32 seed = 3; + } + message RangePartitionPB { message BoundPB { enum Type { @@ -422,33 +466,32 @@ message PartitionPB { // exact string value for the bound. repeated string split_values = 1; } + message RangeWithHashSchemaPB { + // The bounds of this range. + optional RangeBoundPB range_bounds = 1; + // Hash schema for this range. + repeated HashPartitionPB hash_schema = 2; + } // Column names of columns included in the range. All columns must be // a component of the primary key. repeated string columns = 1; - // Range bound. + // Range bounds. repeated RangeBoundPB range_bounds = 2; - // Range split. + // Range splits. repeated SplitValuePB range_splits = 3; + // Ranges with custom hash schemas. + repeated RangeWithHashSchemaPB custom_hash_schema_ranges = 4; } - message HashPartitionPB { - // Column names of columns included in the hash. Every column must be - // a component of the primary key. - repeated string columns = 1; - // Number of buckets into which columns will be hashed. Must be at least 2. - optional int32 num_buckets = 2; - // Seed value for hash calculation. Administrators may set a seed value - // on a per-table basis in order to randomize the mapping of rows to - // buckets. Setting a seed provides some amount of protection against denial - // of service attacks when the hash bucket columns contain user provided - // input. - optional uint32 seed = 3; + // A standalone message representing a hash schema. + message HashSchemaPB { + repeated HashPartitionPB hash_schema = 1; } - // Hash partition message. Support zero or more hash partition levels . + // Table-wide hash schema. repeated HashPartitionPB hash_partitions = 1; - // range partition message. + // Range partitioning information. optional RangePartitionPB range_partition = 2; } @@ -467,17 +510,61 @@ message SchemaPB { // is converted to the PB. Used for creating a new table by kudu tool. message CreateTablePB { optional string table_name = 1; - // Representation of a table's schema, include columns's message and - // primary keys. + // Representation of a table's schema. optional SchemaPB schema = 2; - // The table partition message, include hash partition and range partition. + // Information on the table partitioning. optional PartitionPB partition = 3; - //Number of tablet replica + // Number of replicas for table's tablets. optional int32 num_replicas = 4; // The table's extra configuration properties. optional ExtraConfigPB extra_configs = 5; - // The dimension label for tablets that were created during table creation. Used for - // dimension-specific placement of tablet replicas corresponding to the partitions of - // the newly created table. + // The dimension label for tablets that were created during table creation. + // Used for dimension-specific placement of tablet replicas corresponding + // to the partitions of the newly created table. optional string dimension_label = 6; + // The owner for the newly created table. If not specified, the owner is + // automatically set to the effective OS user name that the kudu CLI tool is + // run with. + optional string owner = 7; + // Table's comment. + optional string comment = 8; +} + +message TablesInfoPB { + message ReplicaInfoPB { + optional string role = 1; + optional string uuid = 2; + optional string host_port = 3; + } + + message TabletWithPartitionPB { + optional string tablet_id = 1; + optional string partition_info = 2; + repeated ReplicaInfoPB replica_info = 3; + } + + message TableInfoPB { + optional string name = 1; + optional int32 num_tablets = 2; + optional int32 num_replicas = 3; + optional uint64 live_row_count = 4; + repeated TabletWithPartitionPB tablet_with_partition = 5; + } + + repeated TableInfoPB tables = 1; +} + +message CreateJwtRequestPB { + // The account ID with which a JWT will be created. + optional string account_id = 1; + + // The subject authenticated by this JWT. + optional string subject = 2; + + // Whether or not the returned token should supply a valid key ID. + optional bool is_valid_key = 3; +} + +message CreateJwtResponsePB { + optional string jwt = 1; } diff --git a/src/Knet.Kudu.Client/Protos/kudu/tserver/tserver.proto b/src/Knet.Kudu.Client/Protos/kudu/tserver/tserver.proto index 0f80bef..b6662de 100644 --- a/src/Knet.Kudu.Client/Protos/kudu/tserver/tserver.proto +++ b/src/Knet.Kudu.Client/Protos/kudu/tserver/tserver.proto @@ -137,6 +137,11 @@ message PingRequestPB { message PingResponsePB { } +message AutoIncrementingColumnPB { + // Value of the auto-incrementing counter + optional int64 auto_incrementing_counter = 1; +} + // A batched set of insert/mutate requests. message WriteRequestPB { required bytes tablet_id = 1; @@ -162,6 +167,9 @@ message WriteRequestPB { // The transaction ID associated with this write request, if any. optional int64 txn_id = 7; + + // The auto-incrementing column information + optional AutoIncrementingColumnPB auto_incrementing_column = 8; } message WriteResponsePB { @@ -186,6 +194,14 @@ message WriteResponsePB { // The timestamp chosen by the server for this write. // TODO KUDU-611 propagate timestamps with server signature. optional fixed64 timestamp = 3; + + // The write operation metrics of this RPC. + // This metrics contains the number of successful/unsuccessful operation + // ('successful_inserts' to 'delete_ignore_errors' fields in ResourceMetricsPB message) + // related to the associated write request. + // Additionally, contains 'commit_wait_duration_usec' metric if the responding server is + // a LEADER and the request external_consistency_mode is COMMIT_WAIT. + optional ResourceMetricsPB resource_metrics = 4; } // A list tablets request @@ -405,6 +421,24 @@ message ResourceMetricsPB { optional int64 cpu_user_nanos = 6; // Total elapsed CPU system time in nanoseconds for all scan rpc requests for this scanner. optional int64 cpu_system_nanos = 7; + // Total number of successful INSERT/INSERT_IGNORE operation. + optional int64 successful_inserts = 8; + // Total number of unsuccessful INSERT_IGNORE operation. + optional int64 insert_ignore_errors = 9; + // Total number of successful UPSERT operation. + optional int64 successful_upserts = 10; + // Total number of successful UPDATE/UPDATE_IGNORE operation. + optional int64 successful_updates = 11; + // Total number of unsuccessful UPDATE_IGNORE operation. + optional int64 update_ignore_errors = 12; + // Total number of successful DELETE operation. + optional int64 successful_deletes = 13; + // Total number of unsuccessful DELETE_IGNORE operation. + optional int64 delete_ignore_errors = 14; + // Total observed commit wait duration in microseconds. + optional int64 commit_wait_duration_usec = 15; + // Total number of UPSERT_IGNORE operations with error. + optional int64 upsert_ignore_errors = 16; } message ScanResponsePB {