diff --git a/.mapping.json b/.mapping.json index 34672b616d2..65d6a4fa5ce 100644 --- a/.mapping.json +++ b/.mapping.json @@ -6001,6 +6001,11 @@ "contrib/ydb/library/yql/utils/log/proto/CMakeLists.linux-x86_64.txt":"", "contrib/ydb/library/yql/utils/log/proto/CMakeLists.txt":"", "contrib/ydb/library/yql/utils/log/proto/CMakeLists.windows-x86_64.txt":"", + "contrib/ydb/library/yql/utils/plan/CMakeLists.darwin-x86_64.txt":"", + "contrib/ydb/library/yql/utils/plan/CMakeLists.linux-aarch64.txt":"", + "contrib/ydb/library/yql/utils/plan/CMakeLists.linux-x86_64.txt":"", + "contrib/ydb/library/yql/utils/plan/CMakeLists.txt":"", + "contrib/ydb/library/yql/utils/plan/CMakeLists.windows-x86_64.txt":"", "contrib/ydb/library/yql/utils/threading/CMakeLists.darwin-x86_64.txt":"", "contrib/ydb/library/yql/utils/threading/CMakeLists.linux-aarch64.txt":"", "contrib/ydb/library/yql/utils/threading/CMakeLists.linux-x86_64.txt":"", diff --git a/contrib/ydb/core/base/appdata.cpp b/contrib/ydb/core/base/appdata.cpp index 3d195cf28b2..8e58772a54d 100644 --- a/contrib/ydb/core/base/appdata.cpp +++ b/contrib/ydb/core/base/appdata.cpp @@ -74,6 +74,7 @@ TAppData::TAppData( , DomainsConfigPtr(new NKikimrConfig::TDomainsConfig()) , BootstrapConfigPtr(new NKikimrConfig::TBootstrap()) , AwsCompatibilityConfigPtr(new NKikimrConfig::TAwsCompatibilityConfig()) + , S3ProxyResolverConfigPtr(new NKikimrConfig::TS3ProxyResolverConfig()) , StreamingConfig(*StreamingConfigPtr.get()) , PQConfig(*PQConfigPtr.get()) , PQClusterDiscoveryConfig(*PQClusterDiscoveryConfigPtr.get()) @@ -94,6 +95,7 @@ TAppData::TAppData( , DomainsConfig(*DomainsConfigPtr.get()) , BootstrapConfig(*BootstrapConfigPtr.get()) , AwsCompatibilityConfig(*AwsCompatibilityConfigPtr.get()) + , S3ProxyResolverConfig(*S3ProxyResolverConfigPtr.get()) , KikimrShouldContinue(kikimrShouldContinue) {} diff --git a/contrib/ydb/core/base/appdata_fwd.h b/contrib/ydb/core/base/appdata_fwd.h index 8a0cbd143f9..067ca4e30f7 100644 --- a/contrib/ydb/core/base/appdata_fwd.h +++ b/contrib/ydb/core/base/appdata_fwd.h @@ -59,6 +59,7 @@ namespace NKikimrConfig { class TDomainsConfig; class TBootstrap; class TAwsCompatibilityConfig; + class TS3ProxyResolverConfig; } namespace NKikimrNetClassifier { @@ -201,6 +202,7 @@ struct TAppData { std::unique_ptr DomainsConfigPtr; std::unique_ptr BootstrapConfigPtr; std::unique_ptr AwsCompatibilityConfigPtr; + std::unique_ptr S3ProxyResolverConfigPtr; std::unique_ptr SharedCacheConfigPtr; NKikimrStream::TStreamingConfig& StreamingConfig; @@ -223,6 +225,7 @@ struct TAppData { NKikimrConfig::TDomainsConfig& DomainsConfig; NKikimrConfig::TBootstrap& BootstrapConfig; NKikimrConfig::TAwsCompatibilityConfig& AwsCompatibilityConfig; + NKikimrConfig::TS3ProxyResolverConfig& S3ProxyResolverConfig; bool EnforceUserTokenRequirement = false; bool AllowHugeKeyValueDeletes = true; // delete when all clients limit deletes per request bool EnableKqpSpilling = false; diff --git a/contrib/ydb/core/cms/cluster_info.cpp b/contrib/ydb/core/cms/cluster_info.cpp index de4cce03048..65e0dafa837 100644 --- a/contrib/ydb/core/cms/cluster_info.cpp +++ b/contrib/ydb/core/cms/cluster_info.cpp @@ -795,6 +795,18 @@ ui64 TClusterInfo::AddExternalLocks(const TNotificationInfo ¬ification, const return locks; } +void TClusterInfo::SetHostMarkers(const TString &hostName, const THashSet &markers) { + for (auto node : NodePtrs(hostName)) { + node->Markers.insert(markers.begin(), markers.end()); + } +} + +void TClusterInfo::ResetHostMarkers(const TString &hostName) { + for (auto node : NodePtrs(hostName)) { + node->Markers.clear(); + } +} + void TClusterInfo::ApplyDowntimes(const TDowntimes &downtimes) { for (auto &pr : downtimes.NodeDowntimes) { diff --git a/contrib/ydb/core/cms/cluster_info.h b/contrib/ydb/core/cms/cluster_info.h index 5a02d220ecd..905b316eb9c 100644 --- a/contrib/ydb/core/cms/cluster_info.h +++ b/contrib/ydb/core/cms/cluster_info.h @@ -297,6 +297,7 @@ class TLockableItem : public TThrRefBase { std::list ScheduledLocks; TVector TempLocks; ui64 DeactivatedLocksOrder = Max(); + THashSet Markers; }; using TLockableItemPtr = TIntrusivePtr; @@ -902,6 +903,9 @@ class TClusterInfo : public TThrRefBase { ui64 AddExternalLocks(const TNotificationInfo ¬ification, const TActorContext *ctx); + void SetHostMarkers(const TString &hostName, const THashSet &markers); + void ResetHostMarkers(const TString &hostName); + void ApplyDowntimes(const TDowntimes &downtimes); void UpdateDowntimes(TDowntimes &downtimes, const TActorContext &ctx); diff --git a/contrib/ydb/core/cms/cms.cpp b/contrib/ydb/core/cms/cms.cpp index afb1dcdc4f3..9c353a6384e 100644 --- a/contrib/ydb/core/cms/cms.cpp +++ b/contrib/ydb/core/cms/cms.cpp @@ -253,6 +253,8 @@ void TCms::AdjustInfo(TClusterInfoPtr &info, const TActorContext &ctx) const info->AddLocks(entry.second, &ctx); for (const auto &entry : State->Notifications) info->AddExternalLocks(entry.second, &ctx); + for (const auto &entry : State->HostMarkers) + info->SetHostMarkers(entry.first, entry.second); } namespace { @@ -284,13 +286,20 @@ bool TCms::CheckPermissionRequest(const TPermissionRequest &request, TStatus::UNKNOWN, }); bool allowPartial = request.GetPartialPermissionAllowed(); - bool schedule = request.GetSchedule() && !request.GetDryRun(); + bool schedule = (request.GetSchedule() || request.GetPrepare()) && !request.GetDryRun(); + + if (request.GetPrepare() && request.ActionsSize() > 1) { + response.MutableStatus()->SetCode(TStatus::WRONG_REQUEST); + response.MutableStatus()->SetReason("Cannot prepare more than one action at once"); + return false; + } response.MutableStatus()->SetCode(TStatus::ALLOW); if (schedule) { scheduled.SetUser(request.GetUser()); scheduled.SetPartialPermissionAllowed(allowPartial); - scheduled.SetSchedule(true); + scheduled.SetSchedule(request.GetSchedule()); + scheduled.SetPrepare(request.GetPrepare()); scheduled.SetReason(request.GetReason()); if (request.HasDuration()) scheduled.SetDuration(request.GetDuration()); @@ -331,7 +340,12 @@ bool TCms::CheckPermissionRequest(const TPermissionRequest &request, LOG_DEBUG(ctx, NKikimrServices::CMS, "Checking action: %s", action.ShortDebugString().data()); - if (CheckAction(action, opts, error, ctx)) { + bool prepared = !request.GetPrepare(); + if (!prepared) { + prepared = CheckPrepareHost(action, error); + } + + if (prepared && CheckAction(action, opts, error, ctx)) { LOG_DEBUG(ctx, NKikimrServices::CMS, "Result: ALLOW"); auto *permission = response.AddPermissions(); @@ -482,11 +496,36 @@ bool TCms::CheckAccess(const TString &token, return false; } -bool TCms::CheckAction(const TAction &action, - const TActionOptions &opts, - TErrorInfo &error, - const TActorContext &ctx) const -{ +bool TCms::CheckPrepareHost(const TAction &action, TErrorInfo &error) const { + if (!State->Sentinel) { + error.Code = TStatus::ERROR; + error.Reason = "Cannot prepare host while Sentinel (self heal) is disabled"; + return false; + } + + switch (action.GetType()) { + case TAction::RESTART_SERVICES: + case TAction::SHUTDOWN_HOST: + case TAction::REBOOT_HOST: + break; + default: + error.Code = TStatus::WRONG_REQUEST; + error.Reason = TStringBuilder() << "Cannot prepare host for action: " << action.GetType(); + return false; + } + + for (const auto node : ClusterInfo->HostNodes(action.GetHost())) { + if (!node->VDisks.empty()) { + error.Code = TStatus::DISALLOW_TEMP; + error.Reason = TStringBuilder() << "Host " << action.GetHost() << " is not prepared yet"; + return false; + } + } + + return true; +} + +bool TCms::CheckAction(const TAction &action, const TActionOptions &opts, TErrorInfo &error, const TActorContext &ctx) const { if (!IsActionHostValid(action, error)) return false; @@ -505,12 +544,11 @@ bool TCms::CheckAction(const TAction &action, case TAction::ADD_DEVICES: case TAction::REMOVE_DEVICES: error.Code = TStatus::ERROR; - error.Reason = Sprintf("Unsupported action action '%s'", - TAction::EType_Name(action.GetType()).data()); + error.Reason = TStringBuilder() << "Unsupported action: " << action.GetType(); return false; default: - error.Code = TStatus::WRONG_REQUEST; - error.Reason = Sprintf("Unknown action '%s'", TAction::EType_Name(action.GetType()).data()); + error.Code = TStatus::WRONG_REQUEST; + error.Reason = TStringBuilder() << "Unknown action: " << static_cast(action.GetType()); return false; } } @@ -924,7 +962,10 @@ void TCms::AcceptPermissions(TPermissionResponse &resp, const TString &requestId auto &permission = *resp.MutablePermissions(i); permission.SetId(owner + "-p-" + ToString(State->NextPermissionId++)); State->Permissions.emplace(permission.GetId(), TPermissionInfo(permission, requestId, owner)); - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::CMS, "Accepting permission"); + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::CMS, "Accepting permission" + << ": id# " << permission.GetId() + << ", requestId# " << requestId + << ", owner# " << owner); ClusterInfo->AddLocks(permission, requestId, owner, &ctx); if (!check) { @@ -1120,6 +1161,9 @@ void TCms::AddHostState(const TClusterInfoPtr &clusterInfo, const TNodeInfo &nod host->SetInterconnectPort(node.IcPort); host->SetTimestamp(timestamp.GetValue()); node.Location.Serialize(host->MutableLocation(), false); + for (auto marker : node.Markers) { + host->AddMarkers(marker); + } if (node.State == UP || node.VDisks || node.PDisks) { for (const auto flag : GetEnumAllValues()) { if (!(node.Services & flag)) { @@ -1141,6 +1185,9 @@ void TCms::AddHostState(const TClusterInfoPtr &clusterInfo, const TNodeInfo &nod device->SetName(vdisk.GetDeviceName()); device->SetState(vdisk.State); device->SetTimestamp(timestamp.GetValue()); + for (auto marker : vdisk.Markers) { + device->AddMarkers(marker); + } } for (const auto &pdId : node.PDisks) { @@ -1149,6 +1196,9 @@ void TCms::AddHostState(const TClusterInfoPtr &clusterInfo, const TNodeInfo &nod device->SetName(pdisk.GetDeviceName()); device->SetState(pdisk.State); device->SetTimestamp(timestamp.GetValue()); + for (auto marker : pdisk.Markers) { + device->AddMarkers(marker); + } } } } @@ -1291,10 +1341,17 @@ void TCms::RemoveRequest(TEvCms::TEvManageRequestRequest::TPtr &ev, const TActor resp->Record.MutableStatus()->SetReason("Unknown request " + id); } else { const auto &request = it->second; + if (request.Owner != user) { resp->Record.MutableStatus()->SetCode(TStatus::WRONG_REQUEST); resp->Record.MutableStatus()->SetReason(Sprintf("Request %s doesn't belong to %s", id.data(), user.data())); } + + if (request.Request.GetPrepare() && request.Request.ActionsSize() < 1) { + resp->Record.MutableStatus()->SetCode(TStatus::WRONG_REQUEST); + resp->Record.MutableStatus()->SetReason( + Sprintf("Request %s used to prepare action and cannot be deleted while permission is valid", id.data())); + } } LOG_DEBUG(ctx, NKikimrServices::CMS, "Resulting status: %s %s", @@ -1470,6 +1527,65 @@ void TCms::PersistNodeTenants(TTransactionContext& txc, const TActorContext& ctx } } +TVector TCms::SetHostMarker(const TString &host, NKikimrCms::EMarker marker, TTransactionContext &txc, const TActorContext &ctx) { + if (State->HostMarkers.contains(host)) { + return {}; + } + + AuditLog(ctx, TStringBuilder() << "Add host marker" + << ": host# " << host + << ", marker# " << marker); + + State->HostMarkers[host] = {marker}; + NIceDb::TNiceDb db(txc.DB); + db.Table().Key(host).Update( + NIceDb::TUpdate(TVector{marker}) + ); + + TVector updateMarkers; + if (ClusterInfo) { + for (const auto node : ClusterInfo->HostNodes(host)) { + updateMarkers.push_back({ + .NodeId = node->NodeId, + .Markers = {marker}, + }); + } + } + + return updateMarkers; +} + +TVector TCms::ResetHostMarkers(const TString &host, TTransactionContext &txc, const TActorContext &ctx) { + if (!State->HostMarkers.contains(host)) { + return {}; + } + + AuditLog(ctx, TStringBuilder() << "Reset host markers" + << ": host# " << host); + + State->HostMarkers.erase(host); + NIceDb::TNiceDb db(txc.DB); + db.Table().Key(host).Delete(); + + TVector updateMarkers; + if (ClusterInfo) { + for (const auto node : ClusterInfo->HostNodes(host)) { + updateMarkers.push_back({ + .NodeId = node->NodeId, + .Markers = {}, + }); + } + } + + return updateMarkers; +} + +void TCms::SentinelUpdateHostMarkers(TVector &&updateMarkers, const TActorContext &ctx) { + if (updateMarkers) { + ctx.Send(State->Sentinel, new TEvSentinel::TEvUpdateHostMarkers(std::move(updateMarkers))); + } +} + void TCms::ProcessQueue() { // To avoid getting stuck in the processing queue for too long, @@ -1728,6 +1844,21 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, } } + if (rec.GetPrepare()) { + for (const auto &action : rec.GetActions()) { + if (State->HostMarkers.contains(action.GetHost())) { + return ReplyWithError( + ev, TStatus::WRONG_REQUEST, TStringBuilder() << "Host '" << action.GetHost() << "' is preparing", ctx); + } + for (const auto node : ClusterInfo->HostNodes(action.GetHost())) { + if (State->HostMarkers.contains(ToString(node->NodeId))) { + return ReplyWithError( + ev, TStatus::WRONG_REQUEST, TStringBuilder() << "Node '" << node->NodeId << "' is preparing", ctx); + } + } + } + } + bool ok = CheckPermissionRequest(rec, resp->Record, scheduled.Request, ctx); // Schedule request if required. @@ -1738,7 +1869,7 @@ void TCms::Handle(TEvCms::TEvPermissionRequest::TPtr &ev, resp->Record.SetRequestId(reqId); TAutoPtr copy; - if (scheduled.Request.ActionsSize()) { + if (scheduled.Request.ActionsSize() || scheduled.Request.GetPrepare()) { scheduled.Owner = user; scheduled.Order = State->NextRequestId - 1; scheduled.RequestId = reqId; @@ -1803,14 +1934,14 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) auto order = request.Order; State->ScheduledRequests.erase(it); - if (scheduled.Request.ActionsSize()) { + if (scheduled.Request.ActionsSize() || scheduled.Request.GetPrepare()) { scheduled.Owner = user; scheduled.Order = order; scheduled.RequestId = rec.GetRequestId(); resp->Record.SetRequestId(scheduled.RequestId); copy = new TRequestInfo(scheduled); - State->ScheduledRequests.emplace(scheduled.RequestId, std::move(scheduled)); + State->ScheduledRequests.emplace(rec.GetRequestId(), std::move(scheduled)); } else { scheduled.RequestId = rec.GetRequestId(); scheduled.Owner = user; @@ -1818,7 +1949,7 @@ void TCms::Handle(TEvCms::TEvCheckRequest::TPtr &ev, const TActorContext &ctx) } if (ok) - AcceptPermissions(resp->Record, scheduled.RequestId, user, ctx, true); + AcceptPermissions(resp->Record, rec.GetRequestId(), user, ctx, true); auto handle = new IEventHandle(ev->Sender, SelfId(), resp.Release(), 0, ev->Cookie); Execute(CreateTxStorePermissions(std::move(ev->Release()), handle, user, std::move(copy)), ctx); @@ -1897,6 +2028,7 @@ bool TCms::IsValidNotificationAction(const TAction &action, TInstant time, case TAction::RESTART_SERVICES: return CheckNotificationRestartServices(action, time, error, ctx); case TAction::SHUTDOWN_HOST: + case TAction::REBOOT_HOST: return CheckNotificationShutdownHost(action, time, error, ctx); case TAction::REPLACE_DEVICES: return CheckNotificationReplaceDevices(action, time, error, ctx); @@ -1907,12 +2039,11 @@ bool TCms::IsValidNotificationAction(const TAction &action, TInstant time, case TAction::ADD_DEVICES: case TAction::REMOVE_DEVICES: error.Code = TStatus::ERROR; - error.Reason = Sprintf("Unsupported action action '%s'", - TAction::EType_Name(action.GetType()).data()); + error.Reason = TStringBuilder() << "Unsupported action: " << action.GetType(); return false; default: - error.Code = TStatus::WRONG_REQUEST; - error.Reason = Sprintf("Unknown action '%s'", TAction::EType_Name(action.GetType()).data()); + error.Code = TStatus::WRONG_REQUEST; + error.Reason = TStringBuilder() << "Unknown action: " << static_cast(action.GetType()); return false; } } @@ -2079,7 +2210,7 @@ void TCms::Handle(TEvCms::TEvGetLogTailRequest::TPtr &ev, const TActorContext &c void TCms::Handle(TEvCms::TEvGetSentinelStateRequest::TPtr &ev, const TActorContext &ctx) { - if(State->Sentinel) { + if (State->Sentinel) { ctx.Send(ev->Forward(State->Sentinel)); } else { auto Response = MakeHolder(); diff --git a/contrib/ydb/core/cms/cms_impl.h b/contrib/ydb/core/cms/cms_impl.h index 33081cf013b..cc209217a2c 100644 --- a/contrib/ydb/core/cms/cms_impl.h +++ b/contrib/ydb/core/cms/cms_impl.h @@ -4,6 +4,7 @@ #include "cms.h" #include "config.h" #include "logger.h" +#include "sentinel.h" #include "services.h" #include "walle.h" @@ -78,6 +79,11 @@ class TCms : public TActor, public TTabletExecutedFlat { void PersistNodeTenants(TTransactionContext &txc, const TActorContext &ctx); + using THostMarkers = TEvSentinel::TEvUpdateHostMarkers::THostMarkers; + TVector SetHostMarker(const TString &host, NKikimrCms::EMarker marker, TTransactionContext &txc, const TActorContext &ctx); + TVector ResetHostMarkers(const TString &host, TTransactionContext &txc, const TActorContext &ctx); + void SentinelUpdateHostMarkers(TVector &&updateMarkers, const TActorContext &ctx); + static void AddHostState(const TClusterInfoPtr &clusterInfo, const TNodeInfo &node, NKikimrCms::TClusterStateResponse &resp, TInstant timestamp); private: @@ -294,8 +300,12 @@ class TCms : public TActor, public TTabletExecutedFlat { NKikimrCms::TStatus::ECode &code, TString &error, const TActorContext &ctx); - bool CheckAction(const NKikimrCms::TAction &action, const TActionOptions &options, - TErrorInfo &error, const TActorContext &ctx) const; + bool CheckPrepareHost(const NKikimrCms::TAction &action, + TErrorInfo &error) const; + bool CheckAction(const NKikimrCms::TAction &action, + const TActionOptions &opts, + TErrorInfo &error, + const TActorContext &ctx) const; bool CheckActionShutdownNode(const NKikimrCms::TAction &action, const TActionOptions &options, const TNodeInfo &node, diff --git a/contrib/ydb/core/cms/cms_state.h b/contrib/ydb/core/cms/cms_state.h index d2581b5936a..8a2c33ad955 100644 --- a/contrib/ydb/core/cms/cms_state.h +++ b/contrib/ydb/core/cms/cms_state.h @@ -31,6 +31,7 @@ struct TCmsState : public TAtomicRefCount { THashMap Permissions; THashMap ScheduledRequests; THashMap Notifications; + THashMap> HostMarkers; TDowntimes Downtimes; ui64 NextPermissionId = 0; ui64 NextRequestId = 0; diff --git a/contrib/ydb/core/cms/cms_tx_load_state.cpp b/contrib/ydb/core/cms/cms_tx_load_state.cpp index c77e30157ac..829ce9f9457 100644 --- a/contrib/ydb/core/cms/cms_tx_load_state.cpp +++ b/contrib/ydb/core/cms/cms_tx_load_state.cpp @@ -34,6 +34,7 @@ class TCms::TTxLoadState : public TTransactionBase { auto maintenanceTasksRowset = db.Table().Range().Select(); auto notificationRowset = db.Table().Range().Select(); auto nodeTenantRowset = db.Table().Range().Select(); + auto hostMarkersRowset = db.Table().Range().Select(); auto logRowset = db.Table().Range().Select(); if (!paramRow.IsReady() @@ -42,6 +43,7 @@ class TCms::TTxLoadState : public TTransactionBase { || !walleTaskRowset.IsReady() || !maintenanceTasksRowset.IsReady() || !notificationRowset.IsReady() + || !hostMarkersRowset.IsReady() || !logRowset.IsReady()) return false; @@ -205,6 +207,16 @@ class TCms::TTxLoadState : public TTransactionBase { return false; } + while (!hostMarkersRowset.EndOfSet()) { + TString host = hostMarkersRowset.GetValue(); + TVector markers = hostMarkersRowset.GetValue(); + + state->HostMarkers[host].insert(markers.begin(), markers.end()); + + if (!hostMarkersRowset.Next()) + return false; + } + if (!state->Downtimes.DbLoadState(txc, ctx)) return false; diff --git a/contrib/ydb/core/cms/cms_tx_remove_permissions.cpp b/contrib/ydb/core/cms/cms_tx_remove_permissions.cpp index 31d3c6b4a80..14a6869b5a5 100644 --- a/contrib/ydb/core/cms/cms_tx_remove_permissions.cpp +++ b/contrib/ydb/core/cms/cms_tx_remove_permissions.cpp @@ -6,6 +6,12 @@ namespace NKikimr::NCms { class TCms::TTxRemovePermissions : public TTransactionBase { + void RemoveRequest(NIceDb::TNiceDb &db, const TString &reqId, const TActorContext &ctx, const TString &reason) { + Self->State->ScheduledRequests.erase(reqId); + db.Table().Key(reqId).Delete(); + Self->AuditLog(ctx, reason); + } + public: TTxRemovePermissions(TCms *self, TVector &&ids, THolder req, TAutoPtr resp, bool expired) : TBase(self) @@ -26,17 +32,29 @@ class TCms::TTxRemovePermissions : public TTransactionBase { if (!Self->State->Permissions.contains(id)) continue; - auto requestId = Self->State->Permissions.find(id)->second.RequestId; + const auto &permission = Self->State->Permissions.find(id)->second; + const TString requestId = permission.RequestId; + const TString host = permission.Action.GetHost(); + Self->State->Permissions.erase(id); db.Table().Key(id).Delete(); - if (Expired && Self->State->ScheduledRequests.contains(requestId)) { - Self->State->ScheduledRequests.erase(requestId); - db.Table().Key(requestId).Delete(); - - Self->AuditLog(ctx, TStringBuilder() << "Remove request" - << ": id# " << requestId - << ", reason# " << "permission " << id << " has expired"); + auto it = Self->State->ScheduledRequests.find(requestId); + if (it != Self->State->ScheduledRequests.end()) { + if (Expired) { + RemoveRequest(db, requestId, ctx, TStringBuilder() << "Remove request" + << ": id# " << requestId + << ", reason# " << "permission " << id << " has expired"); + } + + if (it->second.Request.GetPrepare()) { + auto ret = Self->ResetHostMarkers(host, txc, ctx); + std::move(ret.begin(), ret.end(), std::back_inserter(UpdateMarkers)); + + RemoveRequest(db, requestId, ctx, TStringBuilder() << "Remove request" + << ": id# " << requestId + << ", reason# " << "permission " << id << " was removed"); + } } if (Self->State->WalleRequests.contains(requestId)) { @@ -66,6 +84,7 @@ class TCms::TTxRemovePermissions : public TTransactionBase { } Self->RemoveEmptyTasks(ctx); + Self->SentinelUpdateHostMarkers(std::move(UpdateMarkers), ctx); } private: @@ -73,6 +92,7 @@ class TCms::TTxRemovePermissions : public TTransactionBase { TAutoPtr Response; TVector Ids; bool Expired; + TVector UpdateMarkers; }; ITransaction *TCms::CreateTxRemovePermissions(TVector ids, THolder req, TAutoPtr resp, diff --git a/contrib/ydb/core/cms/cms_tx_remove_request.cpp b/contrib/ydb/core/cms/cms_tx_remove_request.cpp index f584979d01e..c2191eec84b 100644 --- a/contrib/ydb/core/cms/cms_tx_remove_request.cpp +++ b/contrib/ydb/core/cms/cms_tx_remove_request.cpp @@ -20,6 +20,16 @@ class TCms::TTxRemoveRequest : public TTransactionBase { bool Execute(TTransactionContext &txc, const TActorContext &ctx) override { LOG_DEBUG(ctx, NKikimrServices::CMS, "TTxRemoveRequest Execute"); + auto it = Self->State->ScheduledRequests.find(Id); + if (it != Self->State->ScheduledRequests.end()) { + if (it->second.Request.GetPrepare()) { + for (const auto &action : it->second.Request.GetActions()) { + auto ret = Self->ResetHostMarkers(action.GetHost(), txc, ctx); + std::move(ret.begin(), ret.end(), std::back_inserter(UpdateMarkers)); + } + } + } + NIceDb::TNiceDb db(txc.DB); db.Table().Key(Id).Delete(); Self->State->ScheduledRequests.erase(Id); @@ -40,12 +50,14 @@ class TCms::TTxRemoveRequest : public TTransactionBase { } Self->RemoveEmptyTasks(ctx); + Self->SentinelUpdateHostMarkers(std::move(UpdateMarkers), ctx); } private: THolder Request; TAutoPtr Response; TString Id; + TVector UpdateMarkers; }; ITransaction *TCms::CreateTxRemoveRequest(const TString &id, THolder req, TAutoPtr resp) { diff --git a/contrib/ydb/core/cms/cms_tx_store_permissions.cpp b/contrib/ydb/core/cms/cms_tx_store_permissions.cpp index 069b7225984..d0967340fdf 100644 --- a/contrib/ydb/core/cms/cms_tx_store_permissions.cpp +++ b/contrib/ydb/core/cms/cms_tx_store_permissions.cpp @@ -69,13 +69,18 @@ class TCms::TTxStorePermissions : public TTransactionBase { << ": id# " << id << ", validity# " << TInstant::MicroSeconds(deadline) << ", action# " << actionStr); + + if (Scheduled && Scheduled->Request.GetPrepare()) { + auto ret = Self->SetHostMarker(permission.GetAction().GetHost(), NKikimrCms::MARKER_DISK_FAULTY, txc, ctx); + std::move(ret.begin(), ret.end(), std::back_inserter(UpdateMarkers)); + } } if (Scheduled) { auto &id = Scheduled->RequestId; auto &owner = Scheduled->Owner; - if (Scheduled->Request.ActionsSize()) { + if (Scheduled->Request.ActionsSize() || Scheduled->Request.GetPrepare()) { ui64 order = Scheduled->Order; TString requestStr; google::protobuf::TextFormat::PrintToString(Scheduled->Request, &requestStr); @@ -90,6 +95,13 @@ class TCms::TTxStorePermissions : public TTransactionBase { << ", owner# " << owner << ", order# " << order << ", body# " << requestStr); + + if (Scheduled->Request.GetPrepare()) { + for (const auto &action : Scheduled->Request.GetActions()) { + auto ret = Self->SetHostMarker(action.GetHost(), NKikimrCms::MARKER_DISK_FAULTY, txc, ctx); + std::move(ret.begin(), ret.end(), std::back_inserter(UpdateMarkers)); + } + } } else { db.Table().Key(id).Delete(); @@ -108,6 +120,7 @@ class TCms::TTxStorePermissions : public TTransactionBase { Self->Reply(Request.Get(), Response, ctx); Self->SchedulePermissionsCleanup(ctx); + Self->SentinelUpdateHostMarkers(std::move(UpdateMarkers), ctx); } private: @@ -118,6 +131,7 @@ class TCms::TTxStorePermissions : public TTransactionBase { const TMaybe MaintenanceTaskId; ui64 NextPermissionId; ui64 NextRequestId; + TVector UpdateMarkers; }; ITransaction *TCms::CreateTxStorePermissions(THolder req, TAutoPtr resp, diff --git a/contrib/ydb/core/cms/cms_ut.cpp b/contrib/ydb/core/cms/cms_ut.cpp index 0da306bdc98..d5f16acded1 100644 --- a/contrib/ydb/core/cms/cms_ut.cpp +++ b/contrib/ydb/core/cms/cms_ut.cpp @@ -1684,6 +1684,109 @@ Y_UNIT_TEST_SUITE(TCmsTest) { env.DestroyDefaultCmsPipe(); } + + Y_UNIT_TEST(PrepareHostShouldFailWhileSentinelIsDisabled) + { + TCmsTestEnv env(TTestEnvOpts(8).WithoutSentinel()); + env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(0), 60000000) + ), + TStatus::ERROR + ); + } + + Y_UNIT_TEST(PrepareHostShouldFailOnUnsupportedAction) + { + TCmsTestEnv env(TTestEnvOpts(8).WithSentinel()); + env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::REPLACE_DEVICES, env.GetNodeId(0), 60000000, env.PDiskName(0)) + ), + TStatus::WRONG_REQUEST + ); + } + + Y_UNIT_TEST(PrepareHostShouldFailOnMultipleActions) + { + TCmsTestEnv env(TTestEnvOpts(8).WithSentinel()); + env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(0), 60000000), + MakeAction(TAction::SHUTDOWN_HOST, env.GetNodeId(1), 60000000) + ), + TStatus::WRONG_REQUEST + ); + } + + Y_UNIT_TEST(PrepareHost) + { + auto opts = TTestEnvOpts(8).WithSentinel(); + TCmsTestEnv env(opts); + env.SetLogPriority(NKikimrServices::CMS, NLog::PRI_DEBUG); + + // ok + auto request1 = env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 600000000, "storage") + ), + TStatus::DISALLOW_TEMP + ); + // forbid another prepare request for same host + env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 600000000, "storage") + ), + TStatus::WRONG_REQUEST + ); + + // "move" vdisks + auto& node = TFakeNodeWhiteboardService::Info[env.GetNodeId(0)]; + node.VDisksMoved = true; + node.VDiskStateInfo.clear(); + env.RegenerateBSConfig(TFakeNodeWhiteboardService::Config.MutableResponse()->MutableStatus(0)->MutableBaseConfig(), opts); + + // prepared + auto permission1 = env.CheckRequest("user", request1.GetRequestId(), false, TStatus::ALLOW, 1); + env.CheckRejectRequest("user", request1.GetRequestId(), false, TStatus::WRONG_REQUEST); + env.CheckDonePermission("user", permission1.GetPermissions(0).GetId()); + + // allow immediately + auto request2 = env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 600000000, "storage") + ), + TStatus::ALLOW + ); + UNIT_ASSERT_VALUES_EQUAL(request2.PermissionsSize(), 1); + + // check markers after restart + env.RestartCms(); + env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 600000000, "storage") + ), + TStatus::WRONG_REQUEST + ); + + env.CheckRejectRequest("user", request2.GetRequestId(), false, TStatus::WRONG_REQUEST); + env.CheckDonePermission("user", request2.GetPermissions(0).GetId()); + + // restore vdisks + node.VDisksMoved = false; + env.RegenerateBSConfig(TFakeNodeWhiteboardService::Config.MutableResponse()->MutableStatus(0)->MutableBaseConfig(), opts); + + // prepare + auto request3 = env.CheckPermissionRequest( + MakePermissionRequest(TRequestOptions("user").WithPrepare(), + MakeAction(TAction::RESTART_SERVICES, env.GetNodeId(0), 600000000, "storage") + ), + TStatus::DISALLOW_TEMP + ); + + // reject until prepared + env.CheckRejectRequest("user", request3.GetRequestId()); + } } } diff --git a/contrib/ydb/core/cms/cms_ut_common.cpp b/contrib/ydb/core/cms/cms_ut_common.cpp index 7a1f917fa7e..944936ae991 100644 --- a/contrib/ydb/core/cms/cms_ut_common.cpp +++ b/contrib/ydb/core/cms/cms_ut_common.cpp @@ -187,23 +187,6 @@ void TFakeNodeWhiteboardService::Handle(TEvWhiteboard::TEvSystemStateRequest::TP namespace { -struct TFakeNodeInfo { - struct TVDiskIDComparator { - bool operator ()(const TVDiskID& a, const TVDiskID& b) const { - return std::make_tuple(a.GroupID, a.FailRealm, a.FailDomain, a.VDisk) - < std::make_tuple(b.GroupID, b.FailRealm, b.FailDomain, b.VDisk); - } - }; - - TMap TabletStateInfo; - TMap NodeStateInfo; - TMap PDiskStateInfo; - TMap VDiskStateInfo; - TMap BSGroupStateInfo; - NKikimrWhiteboard::TSystemStateInfo SystemStateInfo; - bool Connected = true; -}; - class TFakeTenantPool : public TActorBootstrapped { public: TVector Tenants; @@ -243,7 +226,6 @@ class TFakeTenantPool : public TActorBootstrapped { void GenerateExtendedInfo(TTestActorRuntime &runtime, NKikimrBlobStorage::TBaseConfig *config, ui32 pdisks, ui32 vdiskPerPdisk = 4, const TNodeTenantsMap &tenants = {}, bool useMirror3dcErasure = false) { - TGuard guard(TFakeNodeWhiteboardService::Mutex); ui32 numNodes = runtime.GetNodeCount(); ui32 numNodeGroups = pdisks * vdiskPerPdisk; ui32 numGroups; @@ -271,14 +253,16 @@ void GenerateExtendedInfo(TTestActorRuntime &runtime, NKikimrBlobStorage::TBaseC group.SetErasureSpecies("none"); } - TFakeNodeWhiteboardService::Info.clear(); for (ui32 nodeIndex = 0; nodeIndex < numNodes; ++nodeIndex) { ui32 nodeId = runtime.GetNodeId(nodeIndex); - auto &node = TFakeNodeWhiteboardService::Info[nodeId]; - node.SystemStateInfo.SetVersion(ToString(GetProgramSvnRevision())); - node.SystemStateInfo.SetStartTime(now.GetValue()); - node.SystemStateInfo.SetChangeTime(now.GetValue()); + auto ret = TFakeNodeWhiteboardService::Info.emplace(nodeId, TFakeNodeInfo()); + auto &node = ret.first->second; + if (ret.second) { + node.SystemStateInfo.SetVersion(ToString(GetProgramSvnRevision())); + node.SystemStateInfo.SetStartTime(now.GetValue()); + node.SystemStateInfo.SetChangeTime(now.GetValue()); + } if (tenants.contains(nodeIndex)) { node.SystemStateInfo.AddRoles("Tenant"); @@ -312,6 +296,10 @@ void GenerateExtendedInfo(TTestActorRuntime &runtime, NKikimrBlobStorage::TBaseC pdiskConfig.SetGuid(1); pdiskConfig.SetDriveStatus(NKikimrBlobStorage::ACTIVE); + if (node.VDisksMoved) { + continue; + } + for (ui8 vdiskIndex = 0; vdiskIndex < vdiskPerPdisk; ++vdiskIndex) { ui32 vdiskId = pdiskIndex * vdiskPerPdisk + vdiskIndex; ui32 groupId = groupShift + vdiskId; @@ -400,9 +388,7 @@ static NKikimrConfig::TBootstrap GenerateBootstrapConfig(TTestActorRuntime &runt return res; } -static void SetupServices(TTestActorRuntime &runtime, - const TNodeTenantsMap &tenants) -{ +static void SetupServices(TTestActorRuntime &runtime, const TTestEnvOpts &options) { const ui32 domainsNum = 1; const ui32 disksInDomain = 1; @@ -491,8 +477,8 @@ static void SetupServices(TTestActorRuntime &runtime, runtime.AddLocalService(NNodeWhiteboard::MakeNodeWhiteboardServiceId(runtime.GetNodeId(nodeIndex)), TActorSetupCmd(CreateFakeNodeWhiteboardService(), TMailboxType::Simple, 0), nodeIndex); TVector nodeTenants; - if (tenants.contains(nodeIndex)) - nodeTenants = tenants.at(nodeIndex); + if (options.Tenants.contains(nodeIndex)) + nodeTenants = options.Tenants.at(nodeIndex); runtime.AddLocalService(MakeTenantPoolID(runtime.GetNodeId(nodeIndex)), TActorSetupCmd(new TFakeTenantPool(nodeTenants), TMailboxType::Simple, 0), nodeIndex); } @@ -512,7 +498,7 @@ static void SetupServices(TTestActorRuntime &runtime, runtime.GetAppData().BootstrapConfig = TFakeNodeWhiteboardService::BootstrapConfig; NKikimrCms::TCmsConfig cmsConfig; - cmsConfig.MutableSentinelConfig()->SetEnable(false); + cmsConfig.MutableSentinelConfig()->SetEnable(options.EnableSentinel); runtime.GetAppData().DefaultCmsConfig = MakeHolder(cmsConfig); if (!runtime.IsRealThreads()) { @@ -546,6 +532,8 @@ TCmsTestEnv::TCmsTestEnv(const TTestEnvOpts &options) TFakeNodeWhiteboardService::BootstrapConfig = GenerateBootstrapConfig(*this, options.NodeCount, options.Tenants); + TGuard guard(TFakeNodeWhiteboardService::Mutex); + TFakeNodeWhiteboardService::Info.clear(); GenerateExtendedInfo(*this, config, options.VDisks, 4, options.Tenants, options.UseMirror3dcErasure); SetObserverFunc([](TAutoPtr &event) -> auto { @@ -572,7 +560,7 @@ TCmsTestEnv::TCmsTestEnv(const TTestEnvOpts &options) SetupStateStorage(*this, nodeIndex); } } - SetupServices(*this, options.Tenants); + SetupServices(*this, options); Sender = AllocateEdgeActor(); ClientId = TActorId(); @@ -580,7 +568,7 @@ TCmsTestEnv::TCmsTestEnv(const TTestEnvOpts &options) NKikimrCms::TCmsConfig cmsConfig; cmsConfig.MutableTenantLimits()->SetDisabledNodesRatioLimit(0); cmsConfig.MutableClusterLimits()->SetDisabledNodesRatioLimit(0); - cmsConfig.MutableSentinelConfig()->SetEnable(false); + cmsConfig.MutableSentinelConfig()->SetEnable(options.EnableSentinel); SetCmsConfig(cmsConfig); // Need to allow restart state storage nodes @@ -1170,5 +1158,11 @@ void TCmsTestEnv::EnableNoisyBSCPipe() { TFakeNodeWhiteboardService::NoisyBSCPipe = true; } +void TCmsTestEnv::RegenerateBSConfig(NKikimrBlobStorage::TBaseConfig *config, const TTestEnvOpts &opts) { + TGuard guard(TFakeNodeWhiteboardService::Mutex); + config->Clear(); + GenerateExtendedInfo(*this, config, opts.VDisks, 4, opts.Tenants, opts.UseMirror3dcErasure); +} + } // namespace NCmsTest } // namespace NKikimr diff --git a/contrib/ydb/core/cms/cms_ut_common.h b/contrib/ydb/core/cms/cms_ut_common.h index 2ee324e36cb..ad984f27abc 100644 --- a/contrib/ydb/core/cms/cms_ut_common.h +++ b/contrib/ydb/core/cms/cms_ut_common.h @@ -34,6 +34,7 @@ struct TFakeNodeInfo { TMap VDiskStateInfo; NKikimrWhiteboard::TSystemStateInfo SystemStateInfo; bool Connected = true; + bool VDisksMoved = false; }; class TFakeNodeWhiteboardService : public TActorBootstrapped { @@ -84,6 +85,7 @@ struct TTestEnvOpts { TNodeTenantsMap Tenants; bool UseMirror3dcErasure; bool AdvanceCurrentTime; + bool EnableSentinel; TTestEnvOpts() = default; @@ -99,8 +101,19 @@ struct TTestEnvOpts { , Tenants(tenants) , UseMirror3dcErasure(false) , AdvanceCurrentTime(false) + , EnableSentinel(false) { } + + TTestEnvOpts& WithSentinel() { + EnableSentinel = true; + return *this; + } + + TTestEnvOpts& WithoutSentinel() { + EnableSentinel = false; + return *this; + } }; class TCmsTestEnv : public TTestBasicRuntime { @@ -369,6 +382,8 @@ class TCmsTestEnv : public TTestBasicRuntime { const ui64 CmsId; + void RegenerateBSConfig(NKikimrBlobStorage::TBaseConfig *config, const TTestEnvOpts &opts); + private: void SetupLogging(); diff --git a/contrib/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt b/contrib/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt index 71350c7aefc..5871ec5e3de 100644 --- a/contrib/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt +++ b/contrib/ydb/core/cms/console/CMakeLists.darwin-x86_64.txt @@ -8,11 +8,18 @@ add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -84,3 +91,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + contrib/ydb/core/cms/console/config_item_info.h +) diff --git a/contrib/ydb/core/cms/console/CMakeLists.linux-aarch64.txt b/contrib/ydb/core/cms/console/CMakeLists.linux-aarch64.txt index 1c2c408df2a..d3ab1210176 100644 --- a/contrib/ydb/core/cms/console/CMakeLists.linux-aarch64.txt +++ b/contrib/ydb/core/cms/console/CMakeLists.linux-aarch64.txt @@ -8,12 +8,19 @@ add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -85,3 +92,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + contrib/ydb/core/cms/console/config_item_info.h +) diff --git a/contrib/ydb/core/cms/console/CMakeLists.linux-x86_64.txt b/contrib/ydb/core/cms/console/CMakeLists.linux-x86_64.txt index 1c2c408df2a..d3ab1210176 100644 --- a/contrib/ydb/core/cms/console/CMakeLists.linux-x86_64.txt +++ b/contrib/ydb/core/cms/console/CMakeLists.linux-x86_64.txt @@ -8,12 +8,19 @@ add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -85,3 +92,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + contrib/ydb/core/cms/console/config_item_info.h +) diff --git a/contrib/ydb/core/cms/console/CMakeLists.windows-x86_64.txt b/contrib/ydb/core/cms/console/CMakeLists.windows-x86_64.txt index 71350c7aefc..5871ec5e3de 100644 --- a/contrib/ydb/core/cms/console/CMakeLists.windows-x86_64.txt +++ b/contrib/ydb/core/cms/console/CMakeLists.windows-x86_64.txt @@ -8,11 +8,18 @@ add_subdirectory(util) add_subdirectory(validators) +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) add_library(core-cms-console) target_link_libraries(core-cms-console PUBLIC contrib-libs-cxxsupp yutil + tools-enum_parser-enum_serialization_runtime library-actors-core library-actors-http ydb-core-actorlib_impl @@ -84,3 +91,8 @@ target_sources(core-cms-console PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/tx_processor.cpp ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/util.cpp ) +generate_enum_serilization(core-cms-console + ${CMAKE_SOURCE_DIR}/contrib/ydb/core/cms/console/config_item_info.h + INCLUDE_HEADERS + contrib/ydb/core/cms/console/config_item_info.h +) diff --git a/contrib/ydb/core/cms/console/config_item_info.h b/contrib/ydb/core/cms/console/config_item_info.h new file mode 100644 index 00000000000..a34f64a1307 --- /dev/null +++ b/contrib/ydb/core/cms/console/config_item_info.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +struct TConfigItemInfo { + enum class EUpdateKind { + MutableConfigPartFromFile, + MutableConfigPartFromBaseConfig, + MutableConfigPartMergeFromFile, + ReplaceConfigWithConsoleYaml, + ReplaceConfigWithConsoleProto, + ReplaceConfigWithBase, + LoadYamlConfigFromFile, + SetExplicitly, + UpdateExplicitly, + }; + + struct TUpdate { + const char* File; + ui32 Line; + EUpdateKind Kind; + }; + + TVector Updates; +}; diff --git a/contrib/ydb/core/cms/console/configs_dispatcher.cpp b/contrib/ydb/core/cms/console/configs_dispatcher.cpp index c5523456fd3..7ffec786ca7 100644 --- a/contrib/ydb/core/cms/console/configs_dispatcher.cpp +++ b/contrib/ydb/core/cms/console/configs_dispatcher.cpp @@ -136,7 +136,8 @@ class TConfigsDispatcher : public TActorBootstrapped { const NKikimrConfig::TAppConfig &config, const TMap &labels, const NKikimrConfig::TAppConfig &initialCmsConfig, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig); + const NKikimrConfig::TAppConfig &initialCmsYamlConfig, + const THashMap &configInitInfo); void Bootstrap(); @@ -233,6 +234,7 @@ class TConfigsDispatcher : public TActorBootstrapped { NKikimrConfig::TAppConfig CurrentConfig; const NKikimrConfig::TAppConfig InitialCmsConfig; const NKikimrConfig::TAppConfig InitialCmsYamlConfig; + const THashMap ConfigInitInfo; ui64 NextRequestCookie; TVector HttpRequests; TActorId CommonSubscriptionClient; @@ -256,12 +258,14 @@ TConfigsDispatcher::TConfigsDispatcher( const NKikimrConfig::TAppConfig &config, const TMap &labels, const NKikimrConfig::TAppConfig &initialCmsConfig, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig) + const NKikimrConfig::TAppConfig &initialCmsYamlConfig, + const THashMap &configInitInfo) : Labels(labels) , InitialConfig(config) , CurrentConfig(config) , InitialCmsConfig(initialCmsConfig) , InitialCmsYamlConfig(initialCmsYamlConfig) + , ConfigInitInfo(configInitInfo) , NextRequestCookie(Now().GetValue()) { } @@ -480,143 +484,184 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) } } str << "
" << Endl; - COLLAPSED_REF_CONTENT("state", "State") { - PRE() { - str << "SelfId: " << SelfId() << Endl; - auto s = CurrentStateFunc(); - str << "State: " << ( s == &TThis::StateWork ? "StateWork" - : s == &TThis::StateInit ? "StateInit" - : "Unknown" ) << Endl; - str << "YamlConfigEnabled: " << YamlConfigEnabled << Endl; - str << "Subscriptions: " << Endl; - for (auto &[kinds, subscription] : SubscriptionsByKinds) { - str << "- Kinds: " << KindsToString(kinds) << Endl - << " Subscription: " << Endl - << " Yaml: " << subscription->Yaml << Endl - << " Subscribers: " << Endl; - for (auto &[id, updates] : subscription->Subscribers) { - str << " - Actor: " << id << Endl; - str << " UpdatesSent: " << updates << Endl; - } - if (subscription->YamlVersion) { - str << " YamlVersion: " << subscription->YamlVersion->Version << ".["; - bool first = true; - for (auto &[id, hash] : subscription->YamlVersion->VolatileVersions) { - str << (first ? "" : ",") << id << "." << hash; - first = false; - } - str << "]" << Endl; - } else { - str << " CurrentConfigId: " << subscription->CurrentConfig.Version.ShortDebugString() << Endl; - } - str << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; - if (subscription->UpdateInProcess) { - str << " UpdateInProcess: " << subscription->UpdateInProcess->Record.ShortDebugString() << Endl - << " SubscribersToUpdate:"; - for (auto &id : subscription->SubscribersToUpdate) { - str << " " << id; - } - str << Endl; - str << " UpdateInProcessConfigVersion: " << subscription->UpdateInProcessConfigVersion.ShortDebugString() << Endl - << " UpdateInProcessCookie: " << subscription->UpdateInProcessCookie << Endl; - if (subscription->UpdateInProcessYamlVersion) { - str << " UpdateInProcessYamlVersion: " << subscription->UpdateInProcessYamlVersion->Version << Endl; - } - } - } - str << "Subscribers:" << Endl; - for (auto &[subscriber, _] : SubscriptionsBySubscriber) { - str << "- " << subscriber << Endl; - } + COLLAPSED_REF_CONTENT("effective-config", "Effective config") { + str << "
" << Endl; + str << "It is assumed that all config items marked dynamic are consumed by corresponding components at runtime." << Endl; + str << "
" << Endl; + str << "If any component is unable to update some config at runtime check \"Effective startup config\" below to see actual config for this component." << Endl; + str << "
" << Endl; + str << "Coloring: \"config not set\"," + << " \"config set in dynamic config\", \"config set in static config\"" << Endl; + str << "
" << Endl; + NHttp::OutputRichConfigHTML(str, InitialConfig, YamlProtoConfig, CurrentConfig, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled); + } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("effective-startup-config", "Effective startup config") { + str << "
" << Endl; + str << "Some of these configs may be overwritten by dynamic ones." << Endl; + str << "
" << Endl; + NHttp::OutputConfigHTML(str, InitialConfig); + } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("effective-dynamic-config", "Effective dynamic config") { + str << "
" << Endl; + str << "Some subscribers may get static configs if they exist even if dynamic one of corresponding kind is not present." << Endl; + str << "
" << Endl; + NKikimrConfig::TAppConfig trunc; + if (YamlConfigEnabled) { + ReplaceConfigItems(YamlProtoConfig, trunc, KindsToBitMap(DYNAMIC_KINDS), InitialConfig); + ReplaceConfigItems(CurrentConfig, trunc, KindsToBitMap(NON_YAML_KINDS), trunc, false); + } else { + ReplaceConfigItems(CurrentConfig, trunc, KindsToBitMap(DYNAMIC_KINDS), InitialConfig); } + NHttp::OutputConfigHTML(str, trunc); } str << "
" << Endl; - COLLAPSED_REF_CONTENT("yaml-config", "YAML config") { - DIV() { - TAG(TH5) { - str << "Persistent Config" << Endl; + COLLAPSED_REF_CONTENT("debug-info", "Debug info") { + DIV_CLASS("tab-left") { + COLLAPSED_REF_CONTENT("effective-config-debug-info", "Effective config debug info") { + NHttp::OutputConfigDebugInfoHTML(str, InitialConfig, YamlProtoConfig, CurrentConfig, ConfigInitInfo, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled); } - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-yaml-config"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-yaml-config"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-yaml-config"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } - } - TAG_ATTRS(TDiv, {{"id", "yaml-config-item"}, {"name", "yaml-config-itemm"}}) { - str << YamlConfig; + str << "
" << Endl; + COLLAPSED_REF_CONTENT("state", "State") { + PRE() { + str << "SelfId: " << SelfId() << Endl; + auto s = CurrentStateFunc(); + str << "State: " << ( s == &TThis::StateWork ? "StateWork" + : s == &TThis::StateInit ? "StateInit" + : "Unknown" ) << Endl; + str << "YamlConfigEnabled: " << YamlConfigEnabled << Endl; + str << "Subscriptions: " << Endl; + for (auto &[kinds, subscription] : SubscriptionsByKinds) { + str << "- Kinds: " << KindsToString(kinds) << Endl + << " Subscription: " << Endl + << " Yaml: " << subscription->Yaml << Endl + << " Subscribers: " << Endl; + for (auto &[id, updates] : subscription->Subscribers) { + str << " - Actor: " << id << Endl; + str << " UpdatesSent: " << updates << Endl; + } + if (subscription->YamlVersion) { + str << " YamlVersion: " << subscription->YamlVersion->Version << ".["; + bool first = true; + for (auto &[id, hash] : subscription->YamlVersion->VolatileVersions) { + str << (first ? "" : ",") << id << "." << hash; + first = false; + } + str << "]" << Endl; + } else { + str << " CurrentConfigId: " << subscription->CurrentConfig.Version.ShortDebugString() << Endl; + } + str << " CurrentConfig: " << subscription->CurrentConfig.Config.ShortDebugString() << Endl; + if (subscription->UpdateInProcess) { + str << " UpdateInProcess: " << subscription->UpdateInProcess->Record.ShortDebugString() << Endl + << " SubscribersToUpdate:"; + for (auto &id : subscription->SubscribersToUpdate) { + str << " " << id; + } + str << Endl; + str << " UpdateInProcessConfigVersion: " << subscription->UpdateInProcessConfigVersion.ShortDebugString() << Endl + << " UpdateInProcessCookie: " << subscription->UpdateInProcessCookie << Endl; + if (subscription->UpdateInProcessYamlVersion) { + str << " UpdateInProcessYamlVersion: " << subscription->UpdateInProcessYamlVersion->Version << Endl; + } + } + } + str << "Subscribers:" << Endl; + for (auto &[subscriber, _] : SubscriptionsBySubscriber) { + str << "- " << subscriber << Endl; + } } } - str << "
" << Endl; - for (auto &[id, config] : VolatileYamlConfigs) { + str << "
" << Endl; + COLLAPSED_REF_CONTENT("yaml-config", "YAML config") { DIV() { TAG(TH5) { - str << "Volatile Config Id: " << id << Endl; + str << "Persistent Config" << Endl; } TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-yaml-config"}, {"title", "fold"}}) { DIV_CLASS("yaml-sticky-btn") { } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-yaml-config"}, {"title", "unfold"}}) { DIV_CLASS("yaml-sticky-btn") { } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-yaml-config"}, {"title", "copy"}}) { DIV_CLASS("yaml-sticky-btn") { } } - DIV_CLASS("yaml-config-item") { - str << config; + TAG_ATTRS(TDiv, {{"id", "yaml-config-item"}, {"name", "yaml-config-itemm"}}) { + str << YamlConfig; + } + } + str << "
" << Endl; + for (auto &[id, config] : VolatileYamlConfigs) { + DIV() { + TAG(TH5) { + str << "Volatile Config Id: " << id << Endl; + } + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + DIV_CLASS("yaml-config-item") { + str << config; + } + } } } } } - } - } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { - TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { - DIV_CLASS("yaml-sticky-btn") { } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("resolved-yaml-config", "Resolved YAML config") { + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"id", "fold-resolved-yaml-config"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-resolved-yaml-config"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-resolved-yaml-config"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"id", "resolved-yaml-config-item"}, {"name", "resolved-yaml-config-itemm"}}) { + str << ResolvedYamlConfig; + } + } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"id", "unfold-resolved-yaml-config"}, {"title", "unfold"}}) { - DIV_CLASS("yaml-sticky-btn") { } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("resolved-json-config", "Resolved JSON config") { + PRE() { + str << ResolvedJsonConfig << Endl; + } } - TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"id", "copy-resolved-yaml-config"}, {"title", "copy"}}) { - DIV_CLASS("yaml-sticky-btn") { } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("yaml-proto-config", "YAML proto config") { + NHttp::OutputConfigHTML(str, YamlProtoConfig); } - TAG_ATTRS(TDiv, {{"id", "resolved-yaml-config-item"}, {"name", "resolved-yaml-config-itemm"}}) { - str << ResolvedYamlConfig; + str << "
" << Endl; + COLLAPSED_REF_CONTENT("current-config", "Current config") { + NHttp::OutputConfigHTML(str, CurrentConfig); + } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("initial-config", "Initial config") { + NHttp::OutputConfigHTML(str, InitialConfig); + } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") { + NHttp::OutputConfigHTML(str, InitialCmsConfig); + } + str << "
" << Endl; + COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") { + NHttp::OutputConfigHTML(str, InitialCmsYamlConfig); } } } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("resolved-json-config", "Resolved JSON config") { - PRE() { - str << ResolvedJsonConfig << Endl; - } - } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("yaml-proto-config", "YAML proto config") { - NHttp::OutputConfigHTML(str, YamlProtoConfig); - } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("current-config", "Current config") { - NHttp::OutputConfigHTML(str, CurrentConfig); - } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("initial-config", "Initial config") { - NHttp::OutputConfigHTML(str, InitialConfig); - } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") { - NHttp::OutputConfigHTML(str, InitialCmsConfig); - } - str << "
" << Endl; - COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") { - NHttp::OutputConfigHTML(str, InitialCmsYamlConfig); - } } } } @@ -955,9 +1000,10 @@ IActor *CreateConfigsDispatcher( const NKikimrConfig::TAppConfig &config, const TMap &labels, const NKikimrConfig::TAppConfig &initialCmsConfig, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig) + const NKikimrConfig::TAppConfig &initialCmsYamlConfig, + const THashMap &configInitInfo) { - return new TConfigsDispatcher(config, labels, initialCmsConfig, initialCmsYamlConfig); + return new TConfigsDispatcher(config, labels, initialCmsConfig, initialCmsYamlConfig, configInitInfo); } } // namespace NKikimr::NConsole diff --git a/contrib/ydb/core/cms/console/configs_dispatcher.h b/contrib/ydb/core/cms/console/configs_dispatcher.h index ee53e135ce3..6e7c754832b 100644 --- a/contrib/ydb/core/cms/console/configs_dispatcher.h +++ b/contrib/ydb/core/cms/console/configs_dispatcher.h @@ -2,6 +2,7 @@ #include "defs.h" #include +#include #include @@ -115,7 +116,8 @@ IActor *CreateConfigsDispatcher( const NKikimrConfig::TAppConfig &config, const TMap &labels, const NKikimrConfig::TAppConfig &initialCmsConfig = {}, - const NKikimrConfig::TAppConfig &initialCmsYamlConfig = {}); + const NKikimrConfig::TAppConfig &initialCmsYamlConfig = {}, + const THashMap &configInitInfo = {}); inline TActorId MakeConfigsDispatcherID(ui32 node = 0) { char x[12] = { 'c', 'o', 'n', 'f', 'i', 'g', 's', 'd', 'i', 's', 'p' }; diff --git a/contrib/ydb/core/cms/console/http.cpp b/contrib/ydb/core/cms/console/http.cpp index 43419c63727..b8960053b5f 100644 --- a/contrib/ydb/core/cms/console/http.cpp +++ b/contrib/ydb/core/cms/console/http.cpp @@ -14,6 +14,12 @@ void OutputStyles(IOutputStream &os) << ".tab-left {" << Endl << " margin-left: 20px" << Endl << "}" << Endl + << ".collapse-ref.empty {" << Endl + << " color: red !important" << Endl + << "}" << Endl + << ".collapse-ref.dynamic {" << Endl + << " color: green !important" << Endl + << "}" << Endl << "" << Endl; } @@ -29,6 +35,64 @@ void OutputStaticPart(IOutputStream &os) } } +void PrintField(IOutputStream &os, const NKikimrConfig::TAppConfig &config, const google::protobuf::FieldDescriptor *field) { + auto *reflection = config.GetReflection(); + HTML(os) { + PRE() { + switch (field->cpp_type()) { + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: + os << reflection->GetInt32(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: + os << reflection->GetInt64(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: + os << reflection->GetUInt32(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: + os << reflection->GetUInt64(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + os << reflection->GetDouble(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: + os << reflection->GetFloat(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: + os << reflection->GetBool(config, field); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: + os << reflection->GetEnum(config, field)->name(); + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: + if (field->is_repeated()) { + int count = reflection->FieldSize(config, field); + for (int index = 0; index < count; ++index) { + os << "[" << index << "]: " + << reflection->GetRepeatedString(config, field, index) << '\n'; + } + } else { + os << reflection->GetString(config, field); + } + break; + case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + if (field->is_repeated()) { + int count = reflection->FieldSize(config, field); + for (int index = 0; index < count; ++index) { + os << "[" << index << "]:" << '\n' + << reflection->GetRepeatedMessage(config, field, index).DebugString(); + } + } else { + os << reflection->GetMessage(config, field).DebugString(); + } + break; + default: + os << "<unsupported value type>"; + } + } + } +}; + void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config) { HTML(os) { @@ -41,57 +105,119 @@ void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config TString id = TStringBuilder() << field->name() << "-" << RandomNumber(); COLLAPSED_REF_CONTENT(id, field->name()) { DIV_CLASS("tab-left") { - PRE() { - switch (field->cpp_type()) { - case ::google::protobuf::FieldDescriptor::CPPTYPE_INT32: - os << reflection->GetInt32(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_INT64: - os << reflection->GetInt64(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT32: - os << reflection->GetUInt32(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_UINT64: - os << reflection->GetUInt64(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: - os << reflection->GetDouble(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: - os << reflection->GetFloat(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_BOOL: - os << reflection->GetBool(config, field); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_ENUM: - os << reflection->GetEnum(config, field)->name(); - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_STRING: - if (field->is_repeated()) { - int count = reflection->FieldSize(config, field); - for (int index = 0; index < count; ++index) { - os << "[" << index << "]: " - << reflection->GetRepeatedString(config, field, index) << '\n'; + PrintField(os, config, field); + } + } + os << "
" << Endl; + } + } + } +} + +void OutputConfigDebugInfoHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashMap& configInitInfo, + const THashSet &dynamicKinds, + const THashSet &nonYamlKinds, + bool yamlEnabled) +{ + HTML(os) { + DIV_CLASS("tab-left") { + auto *desc = initialConfig.GetDescriptor(); + auto *reflection = initialConfig.GetReflection(); + + for (int i = 0; i < desc->field_count(); i++) { + auto *field = desc->field(i); + auto tag = field->number(); + + bool yaml = false; + bool empty = false; + bool fallback = false; + + const NKikimrConfig::TAppConfig *config = nullptr; + + auto chooseSource = [&](auto& primaryConfig, auto& fallbackConfig) { + if (field->is_repeated()) { + if (reflection->FieldSize(primaryConfig, field)) { + config = &primaryConfig; + } else if (reflection->FieldSize(fallbackConfig, field)) { + config = &fallbackConfig; + fallback = true; + } else { + empty = true; + } + } else { + if (reflection->HasField(primaryConfig, field)) { + config = &primaryConfig; + } else if (reflection->HasField(fallbackConfig, field)) { + config = &fallbackConfig; + fallback = true; + } else { + empty = true; + } + } + }; + + if (dynamicKinds.contains(tag)) { + if (yamlEnabled && !nonYamlKinds.contains(tag)) { + chooseSource(yamlConfig, initialConfig); + yaml = true; + } else { + chooseSource(protoConfig, initialConfig); + } + } else { + fallback = true; + if (field->is_repeated()) { + if (reflection->FieldSize(initialConfig, field)) { + config = &initialConfig; + } else { + empty = true; + } + } else { + if (reflection->HasField(initialConfig, field)) { + config = &initialConfig; + } else { + empty = true; + } + } + } + + TString id = TStringBuilder() << field->name() << "-" << RandomNumber(); + COLLAPSED_REF_CONTENT(id, field->name(), (empty ? "empty" : ""), (yaml ? "yaml" : ""), (fallback || empty ? "" : "dynamic")) { + DIV_CLASS("tab-left") { + if (empty) { + os << "
" << Endl; + os << "This config item isn't set" << Endl; + os << "
" << Endl; + } else { + if (!configInitInfo.empty()) { + os << "
    " << Endl; + if (auto it = configInitInfo.find(tag); it != configInitInfo.end()) { + for (const auto& update : it->second.Updates) { + os << "
  1. " << update.Kind << " at " << update.File << ":" << update.Line << "
    2. "; } - } else { - os << reflection->GetString(config, field); } - break; - case ::google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - if (field->is_repeated()) { - int count = reflection->FieldSize(config, field); - for (int index = 0; index < count; ++index) { - os << "[" << index << "]:" << '\n' - << reflection->GetRepeatedMessage(config, field, index).DebugString(); - } + if (yaml) { + os << "
  2. Overwritten by dynamic yaml config
    3. "; + } else if (!fallback) { + os << "
  3. Overwritten by dynamic proto config
    4. "; + } + os << "
" << Endl; + } else { + os << "
" << Endl; + if (fallback) { + os << "This config item is set in static config"; + } else if (yaml) { + os << "This config item is set in dynamic yaml config"; } else { - os << reflection->GetMessage(config, field).DebugString(); + os << "This config item is set in dynamic proto config"; } - break; - default: - os << "<unsupported value type>"; + os << "
" << Endl; } + PrintField(os, *config, field); } } } @@ -101,4 +227,16 @@ void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config } } +void OutputRichConfigHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashSet &dynamicKinds, + const THashSet &nonYamlKinds, + bool yamlEnabled) +{ + OutputConfigDebugInfoHTML(os, initialConfig, yamlConfig, protoConfig, {}, dynamicKinds, nonYamlKinds, yamlEnabled); +} + } // namespace NKikimr::NConsole::NHttp diff --git a/contrib/ydb/core/cms/console/http.h b/contrib/ydb/core/cms/console/http.h index 51c90c693b9..70c991dfdeb 100644 --- a/contrib/ydb/core/cms/console/http.h +++ b/contrib/ydb/core/cms/console/http.h @@ -2,21 +2,26 @@ #include "defs.h" #include +#include #include #include -#define COLLAPSED_REF_CONTENT(target, text) \ - WITH_SCOPED(tmp, NKikimr::NConsole::NHttp::TCollapsedRef(__stream, target, text)) +#define COLLAPSED_REF_CONTENT(target, text, ...) \ + WITH_SCOPED(tmp, NKikimr::NConsole::NHttp::TCollapsedRef(__stream, target, text __VA_OPT__(,) __VA_ARGS__)) namespace NKikimr::NConsole::NHttp { struct TCollapsedRef { - TCollapsedRef(IOutputStream& str, const TString& target, const TString& text) + template + TCollapsedRef(IOutputStream& str, const TString& target, const TString& text, TClasses... classes) : Str(str) { - Str << "" + Str << "" << text << ""; Str << "
"; } @@ -40,5 +45,23 @@ struct TCollapsedRef { void OutputStaticPart(IOutputStream &os); void OutputStyles(IOutputStream &os); void OutputConfigHTML(IOutputStream &os, const NKikimrConfig::TAppConfig &config); +void OutputRichConfigHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashSet &dynamicKinds, + const THashSet &nonYamlKinds, + bool yamlEnabled); +void OutputConfigDebugInfoHTML( + IOutputStream &os, + const NKikimrConfig::TAppConfig &initialConfig, + const NKikimrConfig::TAppConfig &yamlConfig, + const NKikimrConfig::TAppConfig &protoConfig, + const THashMap& configInitInfo, + const THashSet &dynamicKinds, + const THashSet &nonYamlKinds, + bool yamlEnabled); + } // namespace NKikimr::NConsole::NHttp diff --git a/contrib/ydb/core/cms/console/util.cpp b/contrib/ydb/core/cms/console/util.cpp index 7d399bf0ae4..6587d69c1de 100644 --- a/contrib/ydb/core/cms/console/util.cpp +++ b/contrib/ydb/core/cms/console/util.cpp @@ -51,11 +51,21 @@ TDynBitMap KindsToBitMap(const TVector &kinds) return result; } +TDynBitMap KindsToBitMap(const THashSet &kinds) +{ + TDynBitMap result; + for (auto &kind : kinds) + result.Set(kind); + + return result; +} + void ReplaceConfigItems( const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds, - const NKikimrConfig::TAppConfig &fallback) + const NKikimrConfig::TAppConfig &fallback, + bool cleanupFallback) { NKikimrConfig::TAppConfig fromCopy = from; NKikimrConfig::TAppConfig fallbackCopy = fallback; @@ -78,7 +88,9 @@ void ReplaceConfigItems( if (reflection->HasField(fromCopy, field)) { reflection->ClearField(&fromCopy, field); } - reflection->ClearField(&fallbackCopy, field); + if (cleanupFallback) { + reflection->ClearField(&fallbackCopy, field); + } } } else { reflection->ClearField(&to, field); diff --git a/contrib/ydb/core/cms/console/util.h b/contrib/ydb/core/cms/console/util.h index 768c58b900c..14eb5727981 100644 --- a/contrib/ydb/core/cms/console/util.h +++ b/contrib/ydb/core/cms/console/util.h @@ -17,6 +17,8 @@ TString KindsToString(const TVector &kinds); TDynBitMap KindsToBitMap(const TVector &kinds); +TDynBitMap KindsToBitMap(const THashSet &kinds); + /** * Replace 'kinds' in 'to' from 'from' * repeated items are removed @@ -25,7 +27,8 @@ void ReplaceConfigItems( const NKikimrConfig::TAppConfig &from, NKikimrConfig::TAppConfig &to, const TDynBitMap &kinds, - const NKikimrConfig::TAppConfig &fallback = {}); + const NKikimrConfig::TAppConfig &fallback = {}, + bool cleanupFallback = true); bool CompareConfigs(const NKikimrConfig::TAppConfig &lhs, const NKikimrConfig::TAppConfig &rhs); diff --git a/contrib/ydb/core/cms/console/ya.make b/contrib/ydb/core/cms/console/ya.make index 4c570d76a31..29c874b3392 100644 --- a/contrib/ydb/core/cms/console/ya.make +++ b/contrib/ydb/core/cms/console/ya.make @@ -71,6 +71,8 @@ SRCS( util.h ) +GENERATE_ENUM_SERIALIZATION(config_item_info.h) + PEERDIR( contrib/ydb/library/actors/core contrib/ydb/library/actors/http diff --git a/contrib/ydb/core/cms/sentinel.cpp b/contrib/ydb/core/cms/sentinel.cpp index 20f80a930b3..137c4ad98de 100644 --- a/contrib/ydb/core/cms/sentinel.cpp +++ b/contrib/ydb/core/cms/sentinel.cpp @@ -62,6 +62,11 @@ void TPDiskStatusComputer::AddState(EPDiskState state) { } EPDiskStatus TPDiskStatusComputer::Compute(EPDiskStatus current, TString& reason) const { + if (ForcedStatus) { + reason = "Forced status"; + return *ForcedStatus; + } + if (!StateCounter) { reason = "Uninitialized StateCounter"; return current; @@ -116,6 +121,14 @@ void TPDiskStatusComputer::Reset() { StateCounter = 0; } +void TPDiskStatusComputer::SetForcedStatus(EPDiskStatus status) { + ForcedStatus = status; +} + +void TPDiskStatusComputer::ResetForcedStatus() { + ForcedStatus.Clear(); +} + /// TPDiskStatus TPDiskStatus::TPDiskStatus(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits) @@ -129,13 +142,9 @@ void TPDiskStatus::AddState(EPDiskState state) { TPDiskStatusComputer::AddState(state); } -bool TPDiskStatus::IsChanged(TString& reason) const { - return Current != Compute(Current, reason); -} - bool TPDiskStatus::IsChanged() const { TString unused; - return IsChanged(unused); + return Current != Compute(Current, unused); } void TPDiskStatus::ApplyChanges(TString& reason) { @@ -196,6 +205,13 @@ void TPDiskInfo::AddState(EPDiskState state) { Touch(); } +/// TNodeInfo + +bool TNodeInfo::HasFaultyMarker() const { + return Markers.contains(NKikimrCms::MARKER_DISK_FAULTY) + || Markers.contains(NKikimrCms::MARKER_DISK_BROKEN); +} + /// TClusterMap TClusterMap::TClusterMap(TSentinelState::TPtr state) @@ -412,9 +428,15 @@ class TConfigUpdater: public TUpdaterBaseNodes.clear(); for (const auto& host : record.GetState().GetHosts()) { if (host.HasNodeId() && host.HasLocation() && host.HasName()) { + THashSet markers; + for (auto marker : host.GetMarkers()) { + markers.insert(static_cast(marker)); + } + SentinelState->Nodes.emplace(host.GetNodeId(), TNodeInfo{ .Host = host.GetName(), .Location = NActors::TNodeLocation(host.GetLocation()), + .Markers = std::move(markers), }); } } @@ -862,13 +884,20 @@ class TSentinel: public TActorBootstrapped { const TPDiskID& id = pdisk.first; TPDiskInfo& info = *(pdisk.second); - if (!SentinelState->Nodes.contains(id.NodeId)) { + auto it = SentinelState->Nodes.find(id.NodeId); + if (it == SentinelState->Nodes.end()) { LOG_E("Missing node info" << ": pdiskId# " << id); info.IgnoreReason = NKikimrCms::TPDiskInfo::MISSING_NODE; continue; } + if (it->second.HasFaultyMarker()) { + info.SetForcedStatus(EPDiskStatus::FAULTY); + } else { + info.ResetForcedStatus(); + } + all.AddPDisk(id); if (info.IsChanged()) { if (info.IsNewStatusGood()) { @@ -961,6 +990,18 @@ class TSentinel: public TActorBootstrapped { NTabletPipe::SendData(SelfId(), CmsState->BSControllerPipe, request.Release(), ++SentinelState->ChangeRequestId); } + void Handle(TEvSentinel::TEvUpdateHostMarkers::TPtr& ev) { + for (auto& [nodeId, markers] : ev->Get()->HostMarkers) { + auto it = SentinelState->Nodes.find(nodeId); + if (it == SentinelState->Nodes.end()) { + // markers will be updated upon next ConfigUpdate iteration + continue; + } + + it->second.Markers = std::move(markers); + } + } + void Handle(TEvCms::TEvGetSentinelStateRequest::TPtr& ev) { const auto& reqRecord = ev->Get()->Record; @@ -1192,6 +1233,7 @@ class TSentinel: public TActorBootstrapped { sFunc(TEvSentinel::TEvConfigUpdated, OnConfigUpdated); sFunc(TEvSentinel::TEvUpdateState, UpdateState); sFunc(TEvSentinel::TEvStateUpdated, OnStateUpdated); + hFunc(TEvSentinel::TEvUpdateHostMarkers, Handle); sFunc(TEvSentinel::TEvBSCPipeDisconnected, OnPipeDisconnected); hFunc(TEvCms::TEvGetSentinelStateRequest, Handle); diff --git a/contrib/ydb/core/cms/sentinel.h b/contrib/ydb/core/cms/sentinel.h index f0d1249348f..d5734aef51a 100644 --- a/contrib/ydb/core/cms/sentinel.h +++ b/contrib/ydb/core/cms/sentinel.h @@ -16,6 +16,8 @@ struct TEvSentinel { EvTimeout, EvBSCPipeDisconnected, + EvUpdateHostMarkers, + EvEnd, }; @@ -30,6 +32,20 @@ struct TEvSentinel { struct TEvTimeout: public TEventLocal {}; struct TEvBSCPipeDisconnected: public TEventLocal {}; + struct TEvUpdateHostMarkers: public TEventLocal { + struct THostMarkers { + ui32 NodeId; + THashSet Markers; + }; + + TVector HostMarkers; + + explicit TEvUpdateHostMarkers(TVector&& hostMarkers) + : HostMarkers(std::move(hostMarkers)) + { + } + }; + }; // TEvSentinel IActor* CreateSentinel(TCmsStatePtr state); diff --git a/contrib/ydb/core/cms/sentinel_impl.h b/contrib/ydb/core/cms/sentinel_impl.h index 77d4a116b1a..9fc06e9e881 100644 --- a/contrib/ydb/core/cms/sentinel_impl.h +++ b/contrib/ydb/core/cms/sentinel_impl.h @@ -28,6 +28,9 @@ class TPDiskStatusComputer { void Reset(); + void SetForcedStatus(EPDiskStatus status); + void ResetForcedStatus(); + private: const ui32& DefaultStateLimit; const TLimitsMap& StateLimits; @@ -35,6 +38,7 @@ class TPDiskStatusComputer { EPDiskState State = NKikimrBlobStorage::TPDiskState::Unknown; mutable EPDiskState PrevState = State; ui64 StateCounter; + TMaybe ForcedStatus; }; // TPDiskStatusComputer @@ -43,7 +47,6 @@ class TPDiskStatus: public TPDiskStatusComputer { explicit TPDiskStatus(EPDiskStatus initialStatus, const ui32& defaultStateLimit, const TLimitsMap& stateLimits); void AddState(EPDiskState state); - bool IsChanged(TString& reason) const; bool IsChanged() const; void ApplyChanges(TString& reason); void ApplyChanges(); @@ -104,6 +107,9 @@ struct TPDiskInfo struct TNodeInfo { TString Host; NActors::TNodeLocation Location; + THashSet Markers; + + bool HasFaultyMarker() const; }; struct TConfigUpdaterState { diff --git a/contrib/ydb/core/cms/sentinel_ut.cpp b/contrib/ydb/core/cms/sentinel_ut.cpp index b32fbeefbe4..26478064818 100644 --- a/contrib/ydb/core/cms/sentinel_ut.cpp +++ b/contrib/ydb/core/cms/sentinel_ut.cpp @@ -156,7 +156,7 @@ Y_UNIT_TEST_SUITE(TSentinelBaseTests) { location.SetUnit(ToString(id)); state->ClusterInfo->AddNode(TEvInterconnect::TNodeInfo(id, name, name, name, 10000, TNodeLocation(location)), nullptr); - sentinelState->Nodes[id] = NSentinel::TNodeInfo{name, NActors::TNodeLocation(location)}; + sentinelState->Nodes[id] = NSentinel::TNodeInfo{name, NActors::TNodeLocation(location), {}}; for (ui64 npdisk : xrange(pdisksPerNode)) { NKikimrBlobStorage::TBaseConfig::TPDisk pdisk; diff --git a/contrib/ydb/core/cms/ut_helpers.cpp b/contrib/ydb/core/cms/ut_helpers.cpp index 64625436e60..1a5246157bd 100644 --- a/contrib/ydb/core/cms/ut_helpers.cpp +++ b/contrib/ydb/core/cms/ut_helpers.cpp @@ -1,9 +1,5 @@ #include "ut_helpers.h" -Y_DECLARE_OUT_SPEC(, NKikimrCms::TAction::EType, os, type) { - os << NKikimrCms::TAction::EType_Name(type); -} - Y_DECLARE_OUT_SPEC(, NKikimrWhiteboard::TTabletStateInfo::ETabletState, os, state) { os << NKikimrWhiteboard::TTabletStateInfo::ETabletState_Name(state); } diff --git a/contrib/ydb/core/cms/ut_helpers.h b/contrib/ydb/core/cms/ut_helpers.h index 1e83f67795e..5a6c7bb4ffa 100644 --- a/contrib/ydb/core/cms/ut_helpers.h +++ b/contrib/ydb/core/cms/ut_helpers.h @@ -67,18 +67,49 @@ void AddActions(TAutoPtr &event, const NKiki AddActions(event, actions...); } +struct TRequestOptions { + TString User; + bool Partial; + bool DryRun; + bool Schedule; + bool Prepare; + + explicit TRequestOptions(const TString &user, bool partial, bool dry, bool schedule) + : User(user) + , Partial(partial) + , DryRun(dry) + , Schedule(schedule) + , Prepare(false) + {} + + explicit TRequestOptions(const TString &user) + : TRequestOptions(user, false, false, false) + {} + + TRequestOptions& WithPrepare() { + Prepare = true; + return *this; + } +}; + template -TAutoPtr MakePermissionRequest(const TString &user, bool partial, bool dry, bool schedule, Ts... actions) { +TAutoPtr MakePermissionRequest(const TRequestOptions &opts, Ts... actions) { TAutoPtr event = new NCms::TEvCms::TEvPermissionRequest; - event->Record.SetUser(user); - event->Record.SetPartialPermissionAllowed(partial); - event->Record.SetDryRun(dry); - event->Record.SetSchedule(schedule); + event->Record.SetUser(opts.User); + event->Record.SetPartialPermissionAllowed(opts.Partial); + event->Record.SetDryRun(opts.DryRun); + event->Record.SetSchedule(opts.Schedule); + event->Record.SetPrepare(opts.Prepare); AddActions(event, actions...); return event; } +template +TAutoPtr MakePermissionRequest(const TString &user, bool partial, bool dry, bool schedule, Ts... actions) { + return MakePermissionRequest(TRequestOptions(user, partial, dry, schedule), actions...); +} + inline void AddPermissions(TAutoPtr &ev, const TString &id) { *ev->Record.AddPermissions() = id; } diff --git a/contrib/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/contrib/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 6e2f35092aa..f514cdb4d43 100644 --- a/contrib/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/contrib/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -23,6 +23,26 @@ extern TAutoPtr DummyAllocatorConfig(); namespace NKikimr { namespace NDriverClient { +struct TCallContext { + const char* File; + int Line; +}; + +#define TRACE_CONFIG_CHANGE(CHANGE_CONTEXT, KIND, CHANGE_KIND) \ + RunConfig.ConfigInitInfo[KIND].Updates.emplace_back( \ + TConfigItemInfo::TUpdate{CHANGE_CONTEXT.File, static_cast(CHANGE_CONTEXT.Line), TConfigItemInfo::EUpdateKind:: CHANGE_KIND}) + +#define TRACE_CONFIG_CHANGE_INPLACE(KIND, CHANGE_KIND) \ + RunConfig.ConfigInitInfo[KIND].Updates.emplace_back( \ + TConfigItemInfo::TUpdate{__FILE__, static_cast(__LINE__), TConfigItemInfo::EUpdateKind:: CHANGE_KIND}) + +#define TRACE_CONFIG_CHANGE_INPLACE_T(KIND, CHANGE_KIND) \ + RunConfig.ConfigInitInfo[NKikimrConsole::TConfigItem:: KIND ## Item].Updates.emplace_back( \ + TConfigItemInfo::TUpdate{__FILE__, static_cast(__LINE__), TConfigItemInfo::EUpdateKind:: CHANGE_KIND}) + +#define CALL_CTX() TCallContext{__FILE__, __LINE__} + + constexpr auto NODE_KIND_YDB = "ydb"; constexpr auto NODE_KIND_YQ = "yq"; @@ -275,7 +295,9 @@ class TClientCommandServerBase : public TClientCommand { TProto *MutableConfigPart(TConfig& config, const char *optname, bool (NKikimrConfig::TAppConfig::*hasConfig)() const, const TProto& (NKikimrConfig::TAppConfig::*getConfig)() const, - TProto* (NKikimrConfig::TAppConfig::*mutableConfig)()) { + TProto* (NKikimrConfig::TAppConfig::*mutableConfig)(), + ui32 kind, + TCallContext callCtx) { TProto *res = nullptr; if (!HierarchicalCfg && (AppConfig.*hasConfig)()) { return nullptr; // this field is already provided in AppConfig, so we don't overwrite it @@ -284,9 +306,11 @@ class TClientCommandServerBase : public TClientCommand { if (optname && config.ParseResult->Has(optname)) { const bool success = ParsePBFromFile(config.ParseResult->Get(optname), res = (AppConfig.*mutableConfig)()); Y_ABORT_UNLESS(success); + TRACE_CONFIG_CHANGE(callCtx, kind, MutableConfigPartFromFile); } else if ((BaseConfig.*hasConfig)()) { res = (AppConfig.*mutableConfig)(); res->CopyFrom((BaseConfig.*getConfig)()); + TRACE_CONFIG_CHANGE(callCtx, kind, MutableConfigPartFromBaseConfig); } return res; @@ -294,7 +318,9 @@ class TClientCommandServerBase : public TClientCommand { template TProto *MutableConfigPartMerge(TConfig& config, const char *optname, - TProto* (NKikimrConfig::TAppConfig::*mutableConfig)()) { + TProto* (NKikimrConfig::TAppConfig::*mutableConfig)(), + ui32 kind, + TCallContext callCtx) { TProto *res = nullptr; if (config.ParseResult->Has(optname)) { @@ -303,6 +329,7 @@ class TClientCommandServerBase : public TClientCommand { Y_ABORT_UNLESS(success); res = (AppConfig.*mutableConfig)(); res->MergeFrom(cfg); + TRACE_CONFIG_CHANGE(callCtx, kind, MutableConfigPartMergeFromFile); } return res; @@ -341,12 +368,14 @@ class TClientCommandServerBase : public TClientCommand { TClientCommand::Parse(config); #define OPTION(NAME, FIELD) MutableConfigPart(config, NAME, &NKikimrConfig::TAppConfig::Has##FIELD, \ - &NKikimrConfig::TAppConfig::Get##FIELD, &NKikimrConfig::TAppConfig::Mutable##FIELD) -#define OPTION_MERGE(NAME, FIELD) MutableConfigPartMerge(config, NAME, &NKikimrConfig::TAppConfig::Mutable##FIELD) + &NKikimrConfig::TAppConfig::Get##FIELD, &NKikimrConfig::TAppConfig::Mutable##FIELD, \ + (ui32)NKikimrConsole::TConfigItem:: FIELD ## Item, TCallContext{__FILE__, __LINE__}) +#define OPTION_MERGE(NAME, FIELD) MutableConfigPartMerge(config, NAME, &NKikimrConfig::TAppConfig::Mutable##FIELD, \ + (ui32)NKikimrConsole::TConfigItem:: FIELD ## Item, TCallContext{__FILE__, __LINE__}) OPTION("auth-file", AuthConfig); LoadBaseConfig(config); - LoadYamlConfig(); + LoadYamlConfig(CALL_CTX()); OPTION_MERGE("auth-token-file", AuthConfig); // start memorylog as soon as possible @@ -437,11 +466,12 @@ class TClientCommandServerBase : public TClientCommand { LoadConfigForDynamicNode(); } - LoadYamlConfig(); + LoadYamlConfig(CALL_CTX()); OPTION("sys-file", ActorSystemConfig); if (!AppConfig.HasActorSystemConfig()) { AppConfig.MutableActorSystemConfig()->CopyFrom(*DummyActorSystemConfig()); + TRACE_CONFIG_CHANGE_INPLACE_T(ActorSystemConfig, SetExplicitly); } OPTION("domains-file", DomainsConfig); @@ -463,15 +493,20 @@ class TClientCommandServerBase : public TClientCommand { } // This flag is set per node and we prefer flag over CMS. if (config.ParseResult->Has("syslog-service-tag") - && !AppConfig.GetLogConfig().GetSysLogService()) + && !AppConfig.GetLogConfig().GetSysLogService()) { AppConfig.MutableLogConfig()->SetSysLogService(SysLogServiceTag); + TRACE_CONFIG_CHANGE_INPLACE_T(LogConfig, UpdateExplicitly); + } - if (config.ParseResult->Has("log-file-name")) + if (config.ParseResult->Has("log-file-name")) { AppConfig.MutableLogConfig()->SetBackendFileName(LogFileName); + TRACE_CONFIG_CHANGE_INPLACE_T(LogConfig, UpdateExplicitly); + } if (auto interconnectConfig = OPTION("ic-file", InterconnectConfig)) { if (config.ParseResult->Has("tcp")) { interconnectConfig->SetStartTcp(true); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } } @@ -479,6 +514,7 @@ class TClientCommandServerBase : public TClientCommand { if (auto bootstrapConfig = OPTION("bootstrap-file", BootstrapConfig)) { bootstrapConfig->MutableCompileServiceConfig()->SetInflightLimit(CompileInflightLimit); + TRACE_CONFIG_CHANGE_INPLACE_T(BootstrapConfig, UpdateExplicitly); } OPTION("vdisk-file", VDiskConfig); @@ -509,43 +545,53 @@ class TClientCommandServerBase : public TClientCommand { if (!AppConfig.HasAllocatorConfig()) { AppConfig.MutableAllocatorConfig()->CopyFrom(*DummyAllocatorConfig()); + TRACE_CONFIG_CHANGE_INPLACE_T(AllocatorConfig, UpdateExplicitly); } // apply certificates, if any if (!PathToInterconnectCertFile.Empty()) { AppConfig.MutableInterconnectConfig()->SetPathToCertificateFile(PathToInterconnectCertFile); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } if (!PathToInterconnectPrivateKeyFile.Empty()) { AppConfig.MutableInterconnectConfig()->SetPathToPrivateKeyFile(PathToInterconnectPrivateKeyFile); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } if (!PathToInterconnectCaFile.Empty()) { AppConfig.MutableInterconnectConfig()->SetPathToCaFile(PathToInterconnectCaFile); + TRACE_CONFIG_CHANGE_INPLACE_T(InterconnectConfig, UpdateExplicitly); } if (AppConfig.HasGRpcConfig() && AppConfig.GetGRpcConfig().HasCert()) { AppConfig.MutableGRpcConfig()->SetPathToCertificateFile(AppConfig.GetGRpcConfig().GetCert()); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!PathToGrpcCertFile.Empty()) { AppConfig.MutableGRpcConfig()->SetPathToCertificateFile(PathToGrpcCertFile); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (AppConfig.HasGRpcConfig() && AppConfig.GetGRpcConfig().HasKey()) { AppConfig.MutableGRpcConfig()->SetPathToPrivateKeyFile(AppConfig.GetGRpcConfig().GetKey()); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!PathToGrpcPrivateKeyFile.Empty()) { AppConfig.MutableGRpcConfig()->SetPathToPrivateKeyFile(PathToGrpcPrivateKeyFile); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (AppConfig.HasGRpcConfig() && AppConfig.GetGRpcConfig().HasCA()) { AppConfig.MutableGRpcConfig()->SetPathToCaFile(AppConfig.GetGRpcConfig().GetCA()); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!PathToGrpcCaFile.Empty()) { AppConfig.MutableGRpcConfig()->SetPathToCaFile(PathToGrpcCaFile); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (!AppConfig.HasDomainsConfig()) @@ -575,6 +621,7 @@ class TClientCommandServerBase : public TClientCommand { if (config.ParseResult->Has("suppress-version-check")) { if (AppConfig.HasNameserviceConfig()) { AppConfig.MutableNameserviceConfig()->SetSuppressVersionCheck(true); + TRACE_CONFIG_CHANGE_INPLACE_T(NameserviceConfig, UpdateExplicitly); } else { ythrow yexception() << "--suppress-version-check option is provided without static nameservice config"; } @@ -589,35 +636,48 @@ class TClientCommandServerBase : public TClientCommand { } } - if (!AppConfig.HasMonitoringConfig()) + if (!AppConfig.HasMonitoringConfig()) { AppConfig.MutableMonitoringConfig()->SetMonitoringThreads(MonitoringThreads); - if (!AppConfig.HasRestartsCountConfig() && RestartsCountFile) + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); + } + if (!AppConfig.HasRestartsCountConfig() && RestartsCountFile) { AppConfig.MutableRestartsCountConfig()->SetRestartsCountFile(RestartsCountFile); + TRACE_CONFIG_CHANGE_INPLACE_T(RestartsCountConfig, UpdateExplicitly); + } // Ports and node type are always applied (even if config was loaded from CMS). - if (MonitoringPort) + if (MonitoringPort) { AppConfig.MutableMonitoringConfig()->SetMonitoringPort(MonitoringPort); - if (MonitoringAddress) + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); + } + if (MonitoringAddress) { AppConfig.MutableMonitoringConfig()->SetMonitoringAddress(MonitoringAddress); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); + } if (MonitoringCertificateFile) { TString sslCertificate = TUnbufferedFileInput(MonitoringCertificateFile).ReadAll(); if (!sslCertificate.empty()) { AppConfig.MutableMonitoringConfig()->SetMonitoringCertificate(sslCertificate); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); } else { ythrow yexception() << "invalid ssl certificate file"; } } - if (SqsHttpPort) + if (SqsHttpPort) { AppConfig.MutableSqsConfig()->MutableHttpServerConfig()->SetPort(SqsHttpPort); + TRACE_CONFIG_CHANGE_INPLACE_T(SqsConfig, UpdateExplicitly); + } if (GRpcPort) { auto& conf = *AppConfig.MutableGRpcConfig(); conf.SetStartGRpcProxy(true); conf.SetPort(GRpcPort); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcsPort) { auto& conf = *AppConfig.MutableGRpcConfig(); conf.SetStartGRpcProxy(true); conf.SetSslPort(GRpcsPort); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcPublicHost) { auto& conf = *AppConfig.MutableGRpcConfig(); @@ -627,6 +687,7 @@ class TClientCommandServerBase : public TClientCommand { ext.SetPublicHost(GRpcPublicHost); } } + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcPublicPort) { auto& conf = *AppConfig.MutableGRpcConfig(); @@ -636,6 +697,7 @@ class TClientCommandServerBase : public TClientCommand { ext.SetPublicPort(GRpcPublicPort); } } + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } if (GRpcsPublicPort) { auto& conf = *AppConfig.MutableGRpcConfig(); @@ -645,25 +707,37 @@ class TClientCommandServerBase : public TClientCommand { ext.SetPublicSslPort(GRpcsPublicPort); } } + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } for (const auto& addr : GRpcPublicAddressesV4) { AppConfig.MutableGRpcConfig()->AddPublicAddressesV4(addr); } + if (GRpcPublicAddressesV4.size()) { + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); + } for (const auto& addr : GRpcPublicAddressesV6) { AppConfig.MutableGRpcConfig()->AddPublicAddressesV6(addr); } + if (GRpcPublicAddressesV6.size()) { + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); + } if (GRpcPublicTargetNameOverride) { AppConfig.MutableGRpcConfig()->SetPublicTargetNameOverride(GRpcPublicTargetNameOverride); + TRACE_CONFIG_CHANGE_INPLACE_T(GRpcConfig, UpdateExplicitly); } - if (config.ParseResult->Has("node-type")) + if (config.ParseResult->Has("node-type")) { AppConfig.MutableTenantPoolConfig()->SetNodeType(NodeType); + TRACE_CONFIG_CHANGE_INPLACE_T(TenantPoolConfig, UpdateExplicitly); + } if (config.ParseResult->Has("tenant") && InterconnectPort != DefaultInterconnectPort) { AppConfig.MutableMonitoringConfig()->SetHostLabelOverride(HostAndICPort()); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); } if (config.ParseResult->Has("data-center")) { AppConfig.MutableMonitoringConfig()->SetDataCenter(to_lower(DataCenter)); + TRACE_CONFIG_CHANGE_INPLACE_T(MonitoringConfig, UpdateExplicitly); } if (config.ParseResult->Has("tenant")) { @@ -672,17 +746,20 @@ class TClientCommandServerBase : public TClientCommand { slot.SetTenantName(TenantName); slot.SetIsDynamic(false); RunConfig.TenantName = TenantName; + TRACE_CONFIG_CHANGE_INPLACE_T(TenantPoolConfig, UpdateExplicitly); } else { auto &slot = *AppConfig.MutableTenantPoolConfig()->AddSlots(); slot.SetId("static-slot"); slot.SetTenantName(CanonizePath(DeduceNodeDomain())); slot.SetIsDynamic(false); RunConfig.TenantName = CanonizePath(DeduceNodeDomain()); + TRACE_CONFIG_CHANGE_INPLACE_T(TenantPoolConfig, UpdateExplicitly); } if (config.ParseResult->Has("data-center")) { if (AppConfig.HasFederatedQueryConfig()) { AppConfig.MutableFederatedQueryConfig()->MutableNodesManager()->SetDataCenter(to_lower(DataCenter)); + TRACE_CONFIG_CHANGE_INPLACE_T(FederatedQueryConfig, UpdateExplicitly); } } @@ -740,6 +817,8 @@ class TClientCommandServerBase : public TClientCommand { } messageBusConfig->SetStartTracingBusProxy(!!TracePath); messageBusConfig->SetTracePath(TracePath); + + TRACE_CONFIG_CHANGE_INPLACE_T(MessageBusConfig, UpdateExplicitly); } if (AppConfig.HasDynamicNameserviceConfig()) { @@ -841,7 +920,7 @@ class TClientCommandServerBase : public TClientCommand { return false; } - inline void LoadYamlConfig() { + inline void LoadYamlConfig(TCallContext callCtx) { for(const TString& yamlConfigFile: YamlConfigFiles) { auto yamlConfig = TFileInput(yamlConfigFile); NKikimrConfig::TAppConfig parsedConfig; @@ -864,6 +943,7 @@ class TClientCommandServerBase : public TClientCommand { if (reflection->HasField(parsedConfig, fieldDescriptor)) { reflection->SwapFields(&AppConfig, &parsedConfig, {fieldDescriptor}); + TRACE_CONFIG_CHANGE(callCtx, fieldIdx, ReplaceConfigWithConsoleProto); } } } @@ -1258,8 +1338,10 @@ class TClientCommandServerBase : public TClientCommand { // By now naming config should be loaded and probably replaced with // info from registration response. Don't lose it in case CMS has no // config for naming service. - if (!AppConfig.HasNameserviceConfig()) + if (!AppConfig.HasNameserviceConfig()) { AppConfig.MutableNameserviceConfig()->Swap(appConfig.MutableNameserviceConfig()); + RunConfig.ConfigInitInfo[NKikimrConsole::TConfigItem::NameserviceConfigItem].Updates.pop_back(); + } } bool SaveConfigForNodeToCache(const NKikimrConfig::TAppConfig &appConfig) { @@ -1329,8 +1411,27 @@ class TClientCommandServerBase : public TClientCommand { if (yamlConfig.HasYamlConfigEnabled() && yamlConfig.GetYamlConfigEnabled()) { appConfig = yamlConfig; NYamlConfig::ReplaceUnmanagedKinds(result.GetConfig(), appConfig); + + for (ui32 kind = NKikimrConsole::TConfigItem::EKind_MIN; kind <= NKikimrConsole::TConfigItem::EKind_MAX; kind++) { + if (kind == NKikimrConsole::TConfigItem::Auto || !NKikimrConsole::TConfigItem::EKind_IsValid(kind)) { + continue; + } + if ((kind == NKikimrConsole::TConfigItem::NameserviceConfigItem && appConfig.HasNameserviceConfig()) + || (kind == NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem && appConfig.HasNetClassifierDistributableConfig()) + || (kind == NKikimrConsole::TConfigItem::NamedConfigsItem && appConfig.NamedConfigsSize())) { + TRACE_CONFIG_CHANGE_INPLACE(kind, ReplaceConfigWithConsoleProto); + } else { + TRACE_CONFIG_CHANGE_INPLACE(kind, ReplaceConfigWithConsoleYaml); + } + } } else { appConfig = result.GetConfig(); + for (ui32 kind = NKikimrConsole::TConfigItem::EKind_MIN; kind <= NKikimrConsole::TConfigItem::EKind_MAX; kind++) { + if (kind == NKikimrConsole::TConfigItem::Auto || !NKikimrConsole::TConfigItem::EKind_IsValid(kind)) { + continue; + } + TRACE_CONFIG_CHANGE_INPLACE(kind, ReplaceConfigWithConsoleProto); + } } if (RunConfig.PathToConfigCacheFile) { diff --git a/contrib/ydb/core/driver_lib/run/config.h b/contrib/ydb/core/driver_lib/run/config.h index 801f6400fdd..052c5c87d42 100644 --- a/contrib/ydb/core/driver_lib/run/config.h +++ b/contrib/ydb/core/driver_lib/run/config.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -127,7 +128,6 @@ union TBasicKikimrServicesMask { static_assert(sizeof(TBasicKikimrServicesMask) == 16, "expected sizeof(TBasicKikimrServicesMask) == 16"); - struct TKikimrRunConfig { NKikimrConfig::TAppConfig& AppConfig; ui32 NodeId; @@ -144,6 +144,7 @@ struct TKikimrRunConfig { NKikimrConfig::TAppConfig InitialCmsConfig; NKikimrConfig::TAppConfig InitialCmsYamlConfig; + THashMap ConfigInitInfo; TKikimrRunConfig(NKikimrConfig::TAppConfig& appConfig, ui32 nodeId = 0, const TKikimrScopeId& scopeId = {}); diff --git a/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 80516eeffad..513ccbfc44f 100644 --- a/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2440,11 +2440,12 @@ TConfigsDispatcherInitializer::TConfigsDispatcherInitializer(const TKikimrRunCon , Labels(runConfig.Labels) , InitialCmsConfig(runConfig.InitialCmsConfig) , InitialCmsYamlConfig(runConfig.InitialCmsYamlConfig) + , ConfigInitInfo(runConfig.ConfigInitInfo) { } void TConfigsDispatcherInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels, InitialCmsConfig, InitialCmsYamlConfig); + IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels, InitialCmsConfig, InitialCmsYamlConfig, ConfigInitInfo); setup->LocalServices.push_back(std::pair( NConsole::MakeConfigsDispatcherID(NodeId), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId))); diff --git a/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.h b/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.h index 80a2ba6212c..fcd7b50a0aa 100644 --- a/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/contrib/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -479,6 +479,7 @@ class TConfigsDispatcherInitializer : public IKikimrServicesInitializer { TMap Labels; NKikimrConfig::TAppConfig InitialCmsConfig; NKikimrConfig::TAppConfig InitialCmsYamlConfig; + THashMap ConfigInitInfo; }; class TConfigsCacheInitializer : public IKikimrServicesInitializer { diff --git a/contrib/ydb/core/driver_lib/run/run.cpp b/contrib/ydb/core/driver_lib/run/run.cpp index ff6f166ceb6..41a1a923e8a 100644 --- a/contrib/ydb/core/driver_lib/run/run.cpp +++ b/contrib/ydb/core/driver_lib/run/run.cpp @@ -1093,6 +1093,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) AppData->AwsCompatibilityConfig = runConfig.AppConfig.GetAwsCompatibilityConfig(); } + if (runConfig.AppConfig.HasS3ProxyResolverConfig()) { + AppData->S3ProxyResolverConfig = runConfig.AppConfig.GetS3ProxyResolverConfig(); + } + // setup resource profiles AppData->ResourceProfiles = new TResourceProfiles; if (runConfig.AppConfig.GetBootstrapConfig().ResourceProfilesSize()) diff --git a/contrib/ydb/core/kqp/opt/CMakeLists.darwin-x86_64.txt b/contrib/ydb/core/kqp/opt/CMakeLists.darwin-x86_64.txt index 0598eba1b03..0a04c3679cd 100644 --- a/contrib/ydb/core/kqp/opt/CMakeLists.darwin-x86_64.txt +++ b/contrib/ydb/core/kqp/opt/CMakeLists.darwin-x86_64.txt @@ -30,7 +30,7 @@ target_link_libraries(core-kqp-opt PUBLIC yql-dq-common yql-dq-opt yql-dq-type_ann - providers-s3-expr_nodes + yql-utils-plan core-kqp-provider tools-enum_parser-enum_serialization_runtime ) diff --git a/contrib/ydb/core/kqp/opt/CMakeLists.linux-aarch64.txt b/contrib/ydb/core/kqp/opt/CMakeLists.linux-aarch64.txt index 4885483e07f..89e69f30144 100644 --- a/contrib/ydb/core/kqp/opt/CMakeLists.linux-aarch64.txt +++ b/contrib/ydb/core/kqp/opt/CMakeLists.linux-aarch64.txt @@ -31,7 +31,7 @@ target_link_libraries(core-kqp-opt PUBLIC yql-dq-common yql-dq-opt yql-dq-type_ann - providers-s3-expr_nodes + yql-utils-plan core-kqp-provider tools-enum_parser-enum_serialization_runtime ) diff --git a/contrib/ydb/core/kqp/opt/CMakeLists.linux-x86_64.txt b/contrib/ydb/core/kqp/opt/CMakeLists.linux-x86_64.txt index 4885483e07f..89e69f30144 100644 --- a/contrib/ydb/core/kqp/opt/CMakeLists.linux-x86_64.txt +++ b/contrib/ydb/core/kqp/opt/CMakeLists.linux-x86_64.txt @@ -31,7 +31,7 @@ target_link_libraries(core-kqp-opt PUBLIC yql-dq-common yql-dq-opt yql-dq-type_ann - providers-s3-expr_nodes + yql-utils-plan core-kqp-provider tools-enum_parser-enum_serialization_runtime ) diff --git a/contrib/ydb/core/kqp/opt/CMakeLists.windows-x86_64.txt b/contrib/ydb/core/kqp/opt/CMakeLists.windows-x86_64.txt index 0598eba1b03..0a04c3679cd 100644 --- a/contrib/ydb/core/kqp/opt/CMakeLists.windows-x86_64.txt +++ b/contrib/ydb/core/kqp/opt/CMakeLists.windows-x86_64.txt @@ -30,7 +30,7 @@ target_link_libraries(core-kqp-opt PUBLIC yql-dq-common yql-dq-opt yql-dq-type_ann - providers-s3-expr_nodes + yql-utils-plan core-kqp-provider tools-enum_parser-enum_serialization_runtime ) diff --git a/contrib/ydb/core/kqp/opt/kqp_query_plan.cpp b/contrib/ydb/core/kqp/opt/kqp_query_plan.cpp index dcbf9dd903f..86e883588d8 100644 --- a/contrib/ydb/core/kqp/opt/kqp_query_plan.cpp +++ b/contrib/ydb/core/kqp/opt/kqp_query_plan.cpp @@ -10,9 +10,10 @@ #include #include #include +#include #include #include -#include +#include #include #include @@ -34,6 +35,21 @@ using namespace NClient; namespace { +TString GetNameByReadType(EPlanTableReadType readType) { + switch (readType) { + case EPlanTableReadType::Unspecified: + return "Unspecified"; + case EPlanTableReadType::FullScan: + return "TableFullScan"; + case EPlanTableReadType::Scan: + return "TableRangeScan"; + case EPlanTableReadType::Lookup: + return "TablePointLookup"; + case EPlanTableReadType::MultiLookup: + return "TableMultiLookup"; + } +} + struct TTableRead { EPlanTableReadType Type = EPlanTableReadType::Unspecified; TVector LookupBy; @@ -223,11 +239,6 @@ class TxPlanSerializer { } private: - struct TPredicate { - TVector Args; - TString Body; - }; - struct TOperator { TMap Properties; TSet Inputs; @@ -380,157 +391,6 @@ class TxPlanSerializer { return QueryPlanNodes[SerializerCtx.PlanNodeId]; } - TString ToStr(const TCoDataCtor& data) { - TStringStream out; - EscapeArbitraryAtom(data.Literal().Value(), '"', &out); - return out.Str(); - } - - TString ToStr(const TCoLambda& lambda) { - return PrettyExprStr(lambda.Body()); - } - - TString ToStr(const TCoAsStruct& asStruct) { - TVector args; - for (const auto& kv : asStruct.Args()) { - auto key = PrettyExprStr(TExprBase(kv->Child(0))); - auto value = PrettyExprStr(TExprBase(kv->Child(1))); - - if (!key.empty() && !value.empty()) { - args.push_back(TStringBuilder() << key << ": " << value); - } - } - - return TStringBuilder() << "{" << JoinStrings(std::move(args), ",") << "}"; - } - - TString ToStr(const TCoAsList& asList) { - TVector args; - for (const auto& arg : asList.Args()) { - if (auto str = PrettyExprStr(TExprBase(arg))) { - args.push_back(std::move(str)); - } - } - - return TStringBuilder() << "[" << JoinStrings(std::move(args), ",") << "]"; - } - - TString ToStr(const TCoMember& member) { - auto structName = PrettyExprStr(member.Struct()); - auto memberName = PrettyExprStr(member.Name()); - - if (!structName.empty() && !memberName.empty()) { - return TStringBuilder() << structName << "." << memberName; - } - - return {}; - } - - TString ToStr(const TCoIfPresent& ifPresent) { - /* expected IfPresent with 3 children: - * 0-Optional, 1-PresentHandler, 2-MissingValue */ - if (ifPresent.Ref().ChildrenSize() == 3) { - auto arg = PrettyExprStr(ifPresent.Optional()); - auto pred = ExtractPredicate(ifPresent.PresentHandler()); - - Y_ENSURE(!pred.Args.empty()); - return std::regex_replace(pred.Body.c_str(), - std::regex(pred.Args[0].c_str()), arg.c_str()).data(); - } - - return "..."; - } - - TString ToStr(const TCoExists& exist) { - if (auto str = PrettyExprStr(exist.Optional())) { - return TStringBuilder() << "Exist(" << str << ")"; - } - - return {}; - } - - TString AggrOpToStr(const TExprBase& aggr) { - TVector args; - for (const auto& child : aggr.Ref().Children()) { - if (auto str = PrettyExprStr(TExprBase(child))) { - args.push_back(std::move(str)); - } - } - - return TStringBuilder() << aggr.Ref().Content() << "(" - << JoinStrings(std::move(args), ",") << ")"; - } - - TString BinaryOpToStr(const TExprBase& op) { - auto left = PrettyExprStr(TExprBase(op.Ref().Child(0))); - auto right = PrettyExprStr(TExprBase(op.Ref().Child(1))); - - TStringBuilder str; - str << left; - if (left && right) { - str << " " << op.Ref().Content() << " "; - } - str << right; - - return str; - } - - TString LogicOpToStr(const TExprBase& op) { - TVector args; - for (const auto& child : op.Ref().Children()) { - if (auto str = PrettyExprStr(TExprBase(child))) { - args.push_back(std::move(str)); - } - } - - return JoinStrings(std::move(args), TStringBuilder() << " " << op.Ref().Content() << " "); - } - - TString PrettyExprStr(const TExprBase& expr) { - static const THashMap aggregations = { - {"AggrMin", "MIN"}, - {"AggrMax", "MAX"}, - {"AggrCountUpdate", "COUNT"}, - {"AggrAdd", "SUM"} - }; - - TStringBuilder str; - - if (expr.Maybe()) { - str << expr.Ref().Child(0)->Content(); - } else if (auto data = expr.Maybe()) { - str << ToStr(data.Cast()); - } else if (auto lambda = expr.Maybe()) { - str << ToStr(lambda.Cast()); - } else if (auto asStruct = expr.Maybe()) { - str << ToStr(asStruct.Cast()); - } else if (auto asList = expr.Maybe()) { - str << ToStr(asList.Cast()); - } else if (auto member = expr.Maybe()) { - str << ToStr(member.Cast()); - } else if (auto ifPresent = expr.Maybe()) { - str << ToStr(ifPresent.Cast()); - } else if (auto exist = expr.Maybe()) { - str << ToStr(exist.Cast()); - } else if (expr.Maybe() || expr.Maybe() || expr.Maybe()) { - str << AggrOpToStr(expr); - } else if (aggregations.contains(expr.Ref().Content())) { - str << aggregations.at(expr.Ref().Content()) << "(" - << PrettyExprStr(TExprBase(expr.Ref().Child(0))) << ")"; - } else if (expr.Maybe() || expr.Maybe()) { - str << BinaryOpToStr(expr); - } else if (expr.Maybe() || expr.Maybe() || expr.Maybe()) { - str << LogicOpToStr(expr); - } else if (expr.Maybe() || expr.Maybe() || expr.Maybe() - || expr.Maybe() || expr.Maybe()) { - str << PrettyExprStr(TExprBase(expr.Ref().Child(0))); - } else { - str << expr.Ref().Content(); - } - - return str; - } - void FillConnectionPlanNode(const TDqConnection& connection, TQueryPlanNode& planNode) { planNode.Type = EPlanNodeType::Connection; @@ -706,10 +566,9 @@ class TxPlanSerializer { ranges.AppendValue(rangeDescr); } - // Scan which fixes only few first members of compound primary key were called "Lookup" - // by older explain version. We continue to do so. if (readInfo.LookupBy.size() > 0) { - readInfo.Type = EPlanTableReadType::Lookup; + bool isFullPk = readInfo.LookupBy.size() == tableData.Metadata->KeyColumnNames.size(); + readInfo.Type = isFullPk ? EPlanTableReadType::Lookup : EPlanTableReadType::Scan; } else { readInfo.Type = hasRangeScans ? EPlanTableReadType::Scan : EPlanTableReadType::FullScan; } @@ -743,19 +602,9 @@ class TxPlanSerializer { SerializerCtx.Tables[table].Reads.push_back(readInfo); - if (readInfo.Type == EPlanTableReadType::Scan) { - op.Properties["Name"] = "TableRangeScan"; - AddOperator(planNode, "TableRangeScan", std::move(op)); - } else if (readInfo.Type == EPlanTableReadType::FullScan) { - op.Properties["Name"] = "TableFullScan"; - AddOperator(planNode, "TableFullScan", std::move(op)); - } else if (readInfo.Type == EPlanTableReadType::Lookup) { - op.Properties["Name"] = "TablePointLookup"; - AddOperator(planNode, "TablePointLookup", std::move(op)); - } else { - op.Properties["Name"] = "TableScan"; - AddOperator(planNode, "TableScan", std::move(op)); - } + auto readName = GetNameByReadType(readInfo.Type); + op.Properties["Name"] = readName; + AddOperator(planNode, readName, std::move(op)); } else { const auto table = TString(sourceSettings.Table().Path()); const auto explainPrompt = TKqpReadTableExplainPrompt::Parse(sourceSettings.ExplainPrompt().Cast()); @@ -767,7 +616,7 @@ class TxPlanSerializer { op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - auto rangesDesc = PrettyExprStr(sourceSettings.RangesExpr()); + auto rangesDesc = NPlanUtils::PrettyExprStr(sourceSettings.RangesExpr()); if (rangesDesc == "Void" || explainPrompt.UsedKeyColumns.empty()) { readInfo.Type = EPlanTableReadType::FullScan; @@ -852,18 +701,100 @@ class TxPlanSerializer { AddOptimizerEstimates(op, sourceSettings); - if (readInfo.Type == EPlanTableReadType::FullScan) { - op.Properties["Name"] = "TableFullScan"; - AddOperator(planNode, "TableFullScan", std::move(op)); - } else { - op.Properties["Name"] = "TableRangesScan"; - AddOperator(planNode, "TableRangesScan", std::move(op)); - } + auto readName = GetNameByReadType(readInfo.Type); + op.Properties["Name"] = readName; + AddOperator(planNode, readName, std::move(op)); SerializerCtx.Tables[table].Reads.push_back(std::move(readInfo)); } } + // Try get cluster from data surce or data sink node + TMaybe TryGetCluster(const TExprBase& d) { + if (d.Raw()->ChildrenSize() >= 2) { + TExprBase child(d.Raw()->Child(1)); + auto cluster = child.Maybe(); + if (cluster) { + return cluster.Cast().StringValue(); + } + } + return Nothing(); + } + + TString RemovePathPrefix(TString path) { + const auto& prefix = SerializerCtx.Config->_KqpTablePathPrefix.Get(); + if (prefix && path.StartsWith(*prefix)) { + size_t count = prefix->size(); + if (path.size() > count && path[count] == '/') { + ++count; + } + path.erase(0, count); + } + return path; + } + + void Visit(const TDqSource& source, TQueryPlanNode& stagePlanNode) { + // YDB sources + if (auto settings = source.Settings().Maybe(); settings.IsValid()) { + Visit(settings.Cast(), stagePlanNode); + return; + } + + // Federated providers + TOperator op; + TCoDataSource dataSource = source.DataSource().Cast(); + const TString dataSourceCategory = dataSource.Category().StringValue(); + IDqIntegration* dqIntegration = nullptr; + + { + auto providerIt = SerializerCtx.TypeCtx.DataSourceMap.find(dataSourceCategory); + if (providerIt != SerializerCtx.TypeCtx.DataSourceMap.end()) { + dqIntegration = providerIt->second->GetDqIntegration(); + } + } + + // Common settings that can be overwritten by provider + op.Properties["Name"] = "Read from external data source"; + op.Properties["SourceType"] = dataSourceCategory; + if (auto cluster = TryGetCluster(dataSource)) { + op.Properties["ExternalDataSource"] = RemovePathPrefix(std::move(*cluster)); + } + + if (dqIntegration) { + dqIntegration->FillSourcePlanProperties(source, op.Properties); + } + + AddOperator(stagePlanNode, "Source", op); + } + + void Visit(const TDqSink& sink, TQueryPlanNode& stagePlanNode) { + // Federated providers + TOperator op; + TCoDataSink dataSink = sink.DataSink().Cast(); + const TString dataSinkCategory = dataSink.Category().StringValue(); + IDqIntegration* dqIntegration = nullptr; + + { + auto providerIt = SerializerCtx.TypeCtx.DataSinkMap.find(dataSinkCategory); + if (providerIt != SerializerCtx.TypeCtx.DataSinkMap.end()) { + dqIntegration = providerIt->second->GetDqIntegration(); + } + } + + // Common settings that can be overwritten by provider + op.Properties["Name"] = "Write to external data source"; + op.Properties["SinkType"] = dataSinkCategory; + if (auto cluster = TryGetCluster(dataSink)) { + op.Properties["ExternalDataSource"] = RemovePathPrefix(std::move(*cluster)); + } + + if (dqIntegration) { + dqIntegration->FillSinkPlanProperties(sink, op.Properties); + } + + AddOperator(stagePlanNode, "Sink", op); + } + void Visit(const TExprBase& expr, TQueryPlanNode& planNode) { if (expr.Maybe()) { auto stageGuid = NDq::TDqStageSettings::Parse(expr.Cast()).Id; @@ -907,39 +838,7 @@ class TxPlanSerializer { for (const auto& input : expr.Cast().Inputs()) { if (auto source = input.Maybe()) { - if (auto settings = source.Settings().Maybe(); settings.IsValid()) { - Visit(settings.Cast(), stagePlanNode); - } else if (auto settings = source.Settings().Maybe(); settings.IsValid()) { - TOperator op; - op.Properties["Name"] = S3ProviderName; - op.Properties["Format"] = "raw"; - auto cluster = source.Cast().DataSource().Cast().Cluster().StringValue(); - if (auto pos = cluster.rfind('/'); pos != TString::npos) { - cluster = cluster.substr(pos + 1); - } - op.Properties["Cluster"] = cluster; - AddOperator(stagePlanNode, "Source", op); - } else if (auto settings = source.Settings().Maybe(); settings.IsValid()) { - TOperator op; - op.Properties["Name"] = S3ProviderName; - op.Properties["Format"] = settings.Cast().Format().StringValue(); - auto cluster = source.Cast().DataSource().Cast().Cluster().StringValue(); - if (auto pos = cluster.rfind('/'); pos != TString::npos) { - cluster = cluster.substr(pos + 1); - } - op.Properties["Cluster"] = cluster; - const TStructExprType* fullRowType = settings.Cast().RowType().Ref().GetTypeAnn()->Cast()->GetType()->Cast(); - auto rowTypeItems = fullRowType->GetItems(); - auto& columns = op.Properties["ReadColumns"]; - for (auto& item : rowTypeItems) { - columns.AppendValue(item->GetName()); - } - AddOperator(stagePlanNode, "Source", op); - } else { - TOperator op; - op.Properties["Name"] = source.Cast().DataSource().Cast().Category().StringValue(); - AddOperator(stagePlanNode, "Source", op); - } + Visit(source.Cast(), stagePlanNode); } else { auto inputCn = input.Cast(); @@ -952,10 +851,8 @@ class TxPlanSerializer { if (auto outputs = expr.Cast().Outputs()) { for (auto output : outputs.Cast()) { - if (output.Maybe()) { - TOperator op; - op.Properties["Name"] = output.DataSink().Cast().Category().StringValue(); - AddOperator(stagePlanNode, "Sink", op); + if (auto sink = output.Maybe()) { + Visit(sink.Cast(), stagePlanNode); } } } @@ -1077,8 +974,8 @@ class TxPlanSerializer { ui32 Visit(const TCoCombineCore& combiner, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Aggregate"; - op.Properties["GroupBy"] = PrettyExprStr(combiner.KeyExtractor()); - op.Properties["Aggregation"] = PrettyExprStr(combiner.UpdateHandler()); + op.Properties["GroupBy"] = NPlanUtils::PrettyExprStr(combiner.KeyExtractor()); + op.Properties["Aggregation"] = NPlanUtils::PrettyExprStr(combiner.UpdateHandler()); return AddOperator(planNode, "Aggregate", std::move(op)); } @@ -1086,7 +983,7 @@ class TxPlanSerializer { ui32 Visit(const TCoSort& sort, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Sort"; - op.Properties["SortBy"] = PrettyExprStr(sort.KeySelectorLambda()); + op.Properties["SortBy"] = NPlanUtils::PrettyExprStr(sort.KeySelectorLambda()); return AddOperator(planNode, "Sort", std::move(op)); } @@ -1094,8 +991,8 @@ class TxPlanSerializer { ui32 Visit(const TCoTop& top, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Top"; - op.Properties["TopBy"] = PrettyExprStr(top.KeySelectorLambda()); - op.Properties["Limit"] = PrettyExprStr(top.Count()); + op.Properties["TopBy"] = NPlanUtils::PrettyExprStr(top.KeySelectorLambda()); + op.Properties["Limit"] = NPlanUtils::PrettyExprStr(top.Count()); return AddOperator(planNode, "Top", std::move(op)); } @@ -1103,8 +1000,8 @@ class TxPlanSerializer { ui32 Visit(const TCoTopSort& topSort, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "TopSort"; - op.Properties["TopSortBy"] = PrettyExprStr(topSort.KeySelectorLambda()); - op.Properties["Limit"] = PrettyExprStr(topSort.Count()); + op.Properties["TopSortBy"] = NPlanUtils::PrettyExprStr(topSort.KeySelectorLambda()); + op.Properties["Limit"] = NPlanUtils::PrettyExprStr(topSort.Count()); return AddOperator(planNode, "TopSort", std::move(op)); } @@ -1112,7 +1009,7 @@ class TxPlanSerializer { ui32 Visit(const TCoTake& take, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Limit"; - op.Properties["Limit"] = PrettyExprStr(take.Count()); + op.Properties["Limit"] = NPlanUtils::PrettyExprStr(take.Count()); return AddOperator(planNode, "Limit", std::move(op)); } @@ -1120,7 +1017,7 @@ class TxPlanSerializer { ui32 Visit(const TCoSkip& skip, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Offset"; - op.Properties["Offset"] = PrettyExprStr(skip.Count()); + op.Properties["Offset"] = NPlanUtils::PrettyExprStr(skip.Count()); return AddOperator(planNode, "Offset", std::move(op)); } @@ -1133,7 +1030,7 @@ class TxPlanSerializer { } ui32 Visit(const TCoIterator& iter, TQueryPlanNode& planNode) { - const auto iterValue = PrettyExprStr(iter.List()); + const auto iterValue = NPlanUtils::PrettyExprStr(iter.List()); TOperator op; op.Properties["Name"] = "Iterator"; @@ -1150,7 +1047,7 @@ class TxPlanSerializer { } ui32 Visit(const TCoPartitionByKey& partitionByKey, TQueryPlanNode& planNode) { - const auto inputValue = PrettyExprStr(partitionByKey.Input()); + const auto inputValue = NPlanUtils::PrettyExprStr(partitionByKey.Input()); TOperator op; op.Properties["Name"] = "PartitionByKey"; @@ -1278,22 +1175,11 @@ class TxPlanSerializer { return AddOperator(planNode, name, std::move(op)); } - TPredicate ExtractPredicate(const TCoLambda& expr) { - TPredicate pred; - pred.Args.reserve(expr.Args().Ref().ChildrenSize()); - for (const auto& child : expr.Args().Ref().Children()) { - pred.Args.push_back(PrettyExprStr(TExprBase(child))); - } - - pred.Body = PrettyExprStr(expr.Body()); - return pred; - } - void AddOptimizerEstimates(TOperator& op, const TExprBase& expr) { if (!SerializerCtx.Config->HasOptEnableCostBasedOptimization()) { return; } - + if (auto stats = SerializerCtx.TypeCtx.GetStats(expr.Raw())) { op.Properties["E-Rows"] = stats->Nrows; op.Properties["E-Cost"] = stats->Cost; @@ -1309,13 +1195,13 @@ class TxPlanSerializer { TOperator op; op.Properties["Name"] = "Filter"; - auto pred = ExtractPredicate(filter.Lambda()); + auto pred = NPlanUtils::ExtractPredicate(filter.Lambda()); op.Properties["Predicate"] = pred.Body; AddOptimizerEstimates(op, filter); if (filter.Limit()) { - op.Properties["Limit"] = PrettyExprStr(filter.Limit().Cast()); + op.Properties["Limit"] = NPlanUtils::PrettyExprStr(filter.Limit().Cast()); } return AddOperator(planNode, "Filter", std::move(op)); @@ -1323,12 +1209,29 @@ class TxPlanSerializer { ui32 Visit(const TKqlLookupTableBase& lookup, TQueryPlanNode& planNode) { auto table = TString(lookup.Table().Path().Value()); + auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + + auto lookupKeysType = lookup.LookupKeys().Ref().GetTypeAnn(); + const TTypeAnnotationNode* lookupKeysItemType = nullptr; + if (lookupKeysType->GetKind() == ETypeAnnotationKind::List) { + lookupKeysItemType = lookupKeysType->Cast()->GetItemType(); + } else if (lookupKeysType->GetKind() == ETypeAnnotationKind::Stream) { + lookupKeysItemType = lookupKeysType->Cast()->GetItemType(); + } else { + Y_ENSURE(false, "Unexpected lookup keys type"); + } + + Y_ENSURE(lookupKeysItemType); + Y_ENSURE(lookupKeysItemType->GetKind() == ETypeAnnotationKind::Struct); + auto lookupKeyColumnsCount = lookupKeysItemType->Cast()->GetSize(); + TTableRead readInfo; - readInfo.Type = EPlanTableReadType::Lookup; + readInfo.Type = lookupKeyColumnsCount == tableData.Metadata->KeyColumnNames.size() + ? EPlanTableReadType::Lookup : EPlanTableReadType::Scan; + auto readName = GetNameByReadType(readInfo.Type); TOperator op; - op.Properties["Name"] = "TablePointLookup"; - auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + op.Properties["Name"] = readName; op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; auto& columns = op.Properties["ReadColumns"]; for (auto const& col : lookup.Columns()) { @@ -1340,7 +1243,7 @@ class TxPlanSerializer { SerializerCtx.Tables[table].Reads.push_back(readInfo); planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - return AddOperator(planNode, "TablePointLookup", std::move(op)); + return AddOperator(planNode, readName, std::move(op)); } ui32 Visit(const TKqlReadTableRangesBase& read, TQueryPlanNode& planNode) { @@ -1354,7 +1257,7 @@ class TxPlanSerializer { op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - auto rangesDesc = PrettyExprStr(read.Ranges()); + auto rangesDesc = NPlanUtils::PrettyExprStr(read.Ranges()); if (rangesDesc == "Void" || explainPrompt.UsedKeyColumns.empty()) { readInfo.Type = EPlanTableReadType::FullScan; @@ -1441,14 +1344,9 @@ class TxPlanSerializer { AddOptimizerEstimates(op, read); - ui32 operatorId; - if (readInfo.Type == EPlanTableReadType::FullScan) { - op.Properties["Name"] = "TableFullScan"; - operatorId = AddOperator(planNode, "TableFullScan", std::move(op)); - } else { - op.Properties["Name"] = "TableRangesScan"; - operatorId = AddOperator(planNode, "TableRangesScan", std::move(op)); - } + auto readName = GetNameByReadType(readInfo.Type); + op.Properties["Name"] = readName; + ui32 operatorId = AddOperator(planNode, readName, std::move(op)); SerializerCtx.Tables[table].Reads.push_back(std::move(readInfo)); return operatorId; @@ -1544,10 +1442,9 @@ class TxPlanSerializer { ranges.AppendValue(rangeDescr); } - // Scan which fixes only few first members of compound primary key were called "Lookup" - // by older explain version. We continue to do so. if (readInfo.LookupBy.size() > 0) { - readInfo.Type = EPlanTableReadType::Lookup; + bool isFullPk = readInfo.LookupBy.size() == tableData.Metadata->KeyColumnNames.size(); + readInfo.Type = isFullPk ? EPlanTableReadType::Lookup : EPlanTableReadType::Scan; } else { readInfo.Type = hasRangeScans ? EPlanTableReadType::Scan : EPlanTableReadType::FullScan; } @@ -1581,20 +1478,9 @@ class TxPlanSerializer { AddOptimizerEstimates(op, read); - ui32 operatorId; - if (readInfo.Type == EPlanTableReadType::Scan) { - op.Properties["Name"] = "TableRangeScan"; - operatorId = AddOperator(planNode, "TableRangeScan", std::move(op)); - } else if (readInfo.Type == EPlanTableReadType::FullScan) { - op.Properties["Name"] = "TableFullScan"; - operatorId = AddOperator(planNode, "TableFullScan", std::move(op)); - } else if (readInfo.Type == EPlanTableReadType::Lookup) { - op.Properties["Name"] = "TablePointLookup"; - operatorId = AddOperator(planNode, "TablePointLookup", std::move(op)); - } else { - op.Properties["Name"] = "TableScan"; - operatorId = AddOperator(planNode, "TableScan", std::move(op)); - } + auto readName = GetNameByReadType(readInfo.Type); + op.Properties["Name"] = readName; + ui32 operatorId = AddOperator(planNode, readName, std::move(op)); return operatorId; } @@ -2113,7 +1999,7 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD } } } - + NKqpProto::TKqpStageExtraStats kqpStageStats; if ((*stat)->GetExtra().UnpackTo(&kqpStageStats)) { auto& nodesStats = stats.InsertValue("NodesScanShards", NJson::JSON_ARRAY); diff --git a/contrib/ydb/core/kqp/opt/ya.make b/contrib/ydb/core/kqp/opt/ya.make index 2f3a8c5cd76..00aef79ee00 100644 --- a/contrib/ydb/core/kqp/opt/ya.make +++ b/contrib/ydb/core/kqp/opt/ya.make @@ -23,7 +23,7 @@ PEERDIR( contrib/ydb/library/yql/dq/common contrib/ydb/library/yql/dq/opt contrib/ydb/library/yql/dq/type_ann - contrib/ydb/library/yql/providers/s3/expr_nodes + contrib/ydb/library/yql/utils/plan contrib/ydb/core/kqp/provider ) diff --git a/contrib/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/contrib/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 8083e4f86c9..00f5b92481d 100644 --- a/contrib/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/contrib/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -738,14 +738,6 @@ class TKikimrDataSink : public TDataProviderBase settings.PartitionBy = Build(ctx, node->Pos()).Done(); } - if (!settings.NotNullColumns.IsValid()) { - settings.NotNullColumns = Build(ctx, node->Pos()).Done(); - } - - if (!settings.SerialColumns.IsValid()) { - settings.SerialColumns = Build(ctx, node->Pos()).Done(); - } - auto temporary = settings.Temporary.IsValid() ? settings.Temporary.Cast() : Build(ctx, node->Pos()).Value("false").Done(); @@ -759,8 +751,6 @@ class TKikimrDataSink : public TDataProviderBase .Temporary(temporary) .Columns(settings.Columns.Cast()) .PrimaryKey(settings.PrimaryKey.Cast()) - .NotNullColumns(settings.NotNullColumns.Cast()) - .SerialColumns(settings.SerialColumns.Cast()) .Settings(settings.Other) .Indexes(settings.Indexes.Cast()) .Changefeeds(settings.Changefeeds.Cast()) diff --git a/contrib/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/contrib/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 990c81cc6df..d3fb46ed556 100644 --- a/contrib/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/contrib/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -519,7 +519,7 @@ class TKiSourceCallableExecutionTransformer : public TAsyncCallbackTransformerGetTypeAnn(); const auto kind = resInput->GetTypeAnn()->GetKind(); - const bool data = kind != ETypeAnnotationKind::Flow && kind != ETypeAnnotationKind::Stream && kind != ETypeAnnotationKind::Optional; + const bool data = kind != ETypeAnnotationKind::Flow && kind != ETypeAnnotationKind::Stream; auto node = ctx.WrapByCallableIf(kind != ETypeAnnotationKind::Stream, "ToStream", ctx.WrapByCallableIf(data, "Just", std::move(resInput))); diff --git a/contrib/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/contrib/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index f13710c54af..ac76d07bb66 100644 --- a/contrib/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/contrib/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -123,10 +123,8 @@ {"Index": 9, "Name": "TableSettings", "Type": "TCoNameValueTupleList"}, {"Index": 10, "Name": "Changefeeds", "Type": "TCoChangefeedList"}, {"Index": 11, "Name": "TableType", "Type": "TCoAtom"}, - {"Index": 12, "Name": "NotNullColumns", "Type": "TCoAtomList"}, - {"Index": 13, "Name": "SerialColumns", "Type": "TCoAtomList"}, - {"Index": 14, "Name": "Temporary", "Type": "TCoAtom"}, - {"Index": 15, "Name": "ExistingOk", "Type": "TCoAtom"} + {"Index": 12, "Name": "Temporary", "Type": "TCoAtom"}, + {"Index": 13, "Name": "ExistingOk", "Type": "TCoAtom"} ] }, { diff --git a/contrib/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/contrib/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 811e33a6cc8..5e1f391e16f 100644 --- a/contrib/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/contrib/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -679,16 +679,6 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over meta->TableSettings.PartitionBy.emplace_back(column.Value()); } - THashSet notNullColumns; - for (const auto& column : create.NotNullColumns()) { - notNullColumns.emplace(column.Value()); - } - - THashSet serialColumns; - for(const auto& column : create.SerialColumns()) { - serialColumns.emplace(column.Value()); - } - for (auto item : create.Columns()) { auto columnTuple = item.Cast(); auto nameNode = columnTuple.Item(0).Cast(); @@ -703,20 +693,6 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over auto isOptional = type->GetKind() == ETypeAnnotationKind::Optional; auto actualType = !isOptional ? type : type->Cast()->GetItemType(); - bool notNull; - if (actualType->GetKind() == ETypeAnnotationKind::Pg) { - notNull = notNullColumns.contains(columnName); - } else { - notNull = !isOptional; - } - - auto scIt = serialColumns.find(columnName); - bool isSerial = false; - if (scIt != serialColumns.end()) { - // notNull = true; - isSerial = true; - } - if (actualType->GetKind() != ETypeAnnotationKind::Data && actualType->GetKind() != ETypeAnnotationKind::Pg ) { @@ -736,18 +712,6 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over columnMeta.Name = columnName; columnMeta.Type = GetColumnTypeName(actualType); - if (columnMeta.IsDefaultKindDefined()) { - ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() << "Default setting for " - << columnName << " column is already set: " - << NKikimrKqp::TKqpColumnMetadataProto::EDefaultKind_Name(columnMeta.DefaultKind))); - return TStatus::Error; - } - - if (isSerial) { - columnMeta.DefaultFromSequence = "_serial_column_" + columnMeta.Name; - columnMeta.SetDefaultFromSequence(); - } - if (actualType->GetKind() == ETypeAnnotationKind::Pg) { auto pgTypeId = actualType->Cast()->GetId(); columnMeta.TypeInfo = NKikimr::NScheme::TTypeInfo( @@ -755,7 +719,6 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over NKikimr::NPg::TypeDescFromPgTypeId(pgTypeId) ); } - columnMeta.NotNull = notNull; if (columnTuple.Size() > 2) { const auto& columnConstraints = columnTuple.Item(2).Cast(); @@ -790,6 +753,19 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over columnMeta.SetDefaultFromLiteral(); FillLiteralProto(constraint.Value().Cast(), columnMeta.DefaultFromLiteral); + } else if (constraint.Name().Value() == "serial") { + + if (columnMeta.IsDefaultKindDefined()) { + ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() << "Default setting for " + << columnName << " column is already set: " + << NKikimrKqp::TKqpColumnMetadataProto::EDefaultKind_Name(columnMeta.DefaultKind))); + return TStatus::Error; + } + + columnMeta.DefaultFromSequence = "_serial_column_" + columnMeta.Name; + columnMeta.SetDefaultFromSequence(); + } else if (constraint.Name().Value() == "not_null") { + columnMeta.NotNull = true; } } } diff --git a/contrib/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp b/contrib/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp index f8f62a9c0b5..e486197d880 100644 --- a/contrib/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp +++ b/contrib/ydb/core/kqp/ut/effects/kqp_effects_ut.cpp @@ -509,7 +509,7 @@ Y_UNIT_TEST_SUITE(KqpEffects) { UNIT_ASSERT_VALUES_EQUAL(table["name"], "/Root/Tmp"); auto reads = table["reads"].GetArraySafe(); UNIT_ASSERT_VALUES_EQUAL(reads.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(reads[0]["type"], "Lookup"); + UNIT_ASSERT_VALUES_EQUAL(reads[0]["type"], "Scan"); UNIT_ASSERT_VALUES_EQUAL(reads[0]["columns"].GetArraySafe().size(), 3); } } diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp b/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp new file mode 100644 index 00000000000..d9e7116d126 --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp @@ -0,0 +1,38 @@ +#include "ch_recipe_ut_helpers.h" + +#include +#include + +namespace NTestUtils { + + TString GetChHost() { + return GetEnv("RECIPE_CLICKHOUSE_HOST", "localhost"); + } + + ui32 GetChPort() { + return FromString(GetEnv("RECIPE_CLICKHOUSE_NATIVE_PORT", "1234")); + } + + TString GetChUser() { + return GetEnv("RECIPE_CLICKHOUSE_USER"); + } + + TString GetChPassword() { + return GetEnv("RECIPE_CLICKHOUSE_PASSWORD"); + } + + TString GetChDatabase() { + return "default"; + } + + NClickHouse::TClient CreateClickhouseClient() { + NClickHouse::TClientOptions opt; + opt + .SetHost(GetChHost()) + .SetPort(GetChPort()) + .SetUser(GetChUser()) + .SetPassword(GetChPassword()); + return NClickHouse::TClient(opt); + } + +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.h b/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.h index c690de57f58..c8f573979b5 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.h +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.h @@ -1,36 +1,15 @@ #pragma once -#include -#include - #include -inline const TString GetChHost() { - return GetEnv("RECIPE_CLICKHOUSE_HOST", "localhost"); -} - -inline ui32 GetChPort() { - return FromString(GetEnv("RECIPE_CLICKHOUSE_NATIVE_PORT", "1234")); -} - -inline const TString GetChUser() { - return GetEnv("RECIPE_CLICKHOUSE_USER"); -} +namespace NTestUtils { -inline const TString GetChPassword() { - return GetEnv("RECIPE_CLICKHOUSE_PASSWORD"); -} + TString GetChHost(); + ui32 GetChPort(); + TString GetChUser(); + TString GetChPassword(); + TString GetChDatabase(); -inline const TString GetChDatabase() { - return "default"; -} + NClickHouse::TClient CreateClickhouseClient(); -inline NClickHouse::TClient CreateClickhouseClient() { - NClickHouse::TClientOptions opt; - opt - .SetHost(GetChHost()) - .SetPort(GetChPort()) - .SetUser(GetChUser()) - .SetPassword(GetChPassword()); - return NClickHouse::TClient(opt); -} +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp b/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp new file mode 100644 index 00000000000..23d5768f16f --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp @@ -0,0 +1,35 @@ +#include "connector_recipe_ut_helpers.h" + +#include +#include + +namespace NTestUtils { + + TString GetConnectorHost() { + return GetEnv("YDB_CONNECTOR_RECIPE_GRPC_HOST", "localhost"); + } + + ui32 GetConnectorPort() { + const TString port = GetEnv("YDB_CONNECTOR_RECIPE_GRPC_PORT"); + UNIT_ASSERT_C(port, "No connector port specified"); + return FromString(port); + } + + std::shared_ptr MakeKikimrRunnerWithConnector() { + NYql::TGenericConnectorConfig clientCfg; + clientCfg.MutableEndpoint()->set_host(GetConnectorHost()); + clientCfg.MutableEndpoint()->set_port(GetConnectorPort()); + + NKikimrConfig::TAppConfig appCfg; + appCfg.MutableFeatureFlags()->SetEnableExternalDataSources(true); + + auto kikimr = NKikimr::NKqp::NFederatedQueryTest::MakeKikimrRunner( + NYql::IHTTPGateway::Make(), + NYql::NConnector::MakeClientGRPC(clientCfg), + nullptr, + appCfg); + kikimr->GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + return kikimr; + } + +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.h b/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.h index 6ea689e3aa2..4bb38b65f0f 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.h +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.h @@ -1,12 +1,15 @@ #pragma once +#include +#include +#include -#include -#include +#include -inline const TString GetConnectorHost() { - return GetEnv("YDB_CONNECTOR_RECIPE_GRPC_HOST", "localhost"); -} +namespace NTestUtils { -inline ui32 GetConnectorPort() { - return FromString(GetEnv("YDB_CONNECTOR_RECIPE_GRPC_PORT")); -} + TString GetConnectorHost(); + ui32 GetConnectorPort(); + + std::shared_ptr MakeKikimrRunnerWithConnector(); + +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp b/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp new file mode 100644 index 00000000000..d6b369cc97b --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp @@ -0,0 +1,178 @@ +#include "ch_recipe_ut_helpers.h" +#include "connector_recipe_ut_helpers.h" +#include "pg_recipe_ut_helpers.h" +#include +#include +#include + +#include + +#include +#include + +#include + +using namespace NYdb; +using namespace NYdb::NQuery; +using namespace NTestUtils; +using namespace fmt::literals; + +Y_UNIT_TEST_SUITE(KqpGenericPlanTest) { + Y_UNIT_TEST(PgSource) { + pqxx::connection pgConnection = CreatePostgresqlConnection(); + + { + pqxx::work work{pgConnection}; + const TString sql = R"sql( + CREATE TABLE pg_table_plan_test ( + key INT4 PRIMARY KEY, + name TEXT, + value INT4 + ) + )sql"; + work.exec(sql); + work.commit(); + } + + std::shared_ptr kikimr = MakeKikimrRunnerWithConnector(); + + auto tableCLient = kikimr->GetTableClient(); + auto session = tableCLient.CreateSession().GetValueSync().GetSession(); + + // external tables to pg/ch + { + const TString sql = fmt::format( + R"sql( + CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value=""); + CREATE EXTERNAL DATA SOURCE pg_data_source WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="{pg_host}:{pg_port}", + DATABASE_NAME="{pg_database}", + USE_TLS="FALSE", + AUTH_METHOD="BASIC", + PROTOCOL="NATIVE", + LOGIN="{pg_user}", + PASSWORD_SECRET_NAME="pg_password_obj" + ); + )sql", + "pg_host"_a = GetPgHost(), + "pg_port"_a = GetPgPort(), + "pg_user"_a = GetPgUser(), + "pg_database"_a = GetPgDatabase()); + auto result = session.ExecuteSchemeQuery(sql).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + const TString sql = R"sql( + PRAGMA generic.UsePredicatePushdown="true"; + SELECT * FROM pg_data_source.pg_table_plan_test + WHERE key > 42 AND value <> 0 + )sql"; + + auto queryClient = kikimr->GetQueryClient(); + TExecuteQueryResult queryResult = queryClient.ExecuteQuery( + sql, + TTxControl::BeginTx().CommitTx(), + TExecuteQuerySettings().ExecMode(EExecMode::Explain)) + .GetValueSync(); + + UNIT_ASSERT_C(queryResult.IsSuccess(), queryResult.GetIssues().ToString()); + UNIT_ASSERT(queryResult.GetStats()); + UNIT_ASSERT(queryResult.GetStats()->GetPlan()); + Cerr << "Plan: " << *queryResult.GetStats()->GetPlan() << Endl; + NJson::TJsonValue plan; + UNIT_ASSERT(NJson::ReadJsonTree(*queryResult.GetStats()->GetPlan(), &plan)); + + const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Filter-Source"); + const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[1]; + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "pg_data_source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Database"].GetStringSafe(), GetPgDatabase()); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Protocol"].GetStringSafe(), "Native"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Table"].GetStringSafe(), "pg_table_plan_test"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Read from external data source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["SourceType"].GetStringSafe(), "PostgreSql"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[1].GetStringSafe(), "name"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[2].GetStringSafe(), "value"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Filter"].GetStringSafe(), "item.key > 42 And item.value != 0"); + } + + Y_UNIT_TEST(ChSource) { + NClickHouse::TClient chClient = CreateClickhouseClient(); + + // ch_table_plan_test + { + const TString sql = R"sql( + CREATE TABLE ch_table_plan_test ( + key INT PRIMARY KEY, + name TEXT NULL + ) + ENGINE = MergeTree + )sql"; + chClient.Execute(sql); + } + + std::shared_ptr kikimr = MakeKikimrRunnerWithConnector(); + + auto tableCLient = kikimr->GetTableClient(); + auto session = tableCLient.CreateSession().GetValueSync().GetSession(); + + // external tables to pg/ch + { + const TString sql = fmt::format( + R"sql( + CREATE OBJECT ch_password_obj (TYPE SECRET) WITH (value="{ch_password}"); + CREATE EXTERNAL DATA SOURCE ch_data_source WITH ( + SOURCE_TYPE="ClickHouse", + LOCATION="{ch_host}:{ch_port}", + DATABASE_NAME="{ch_database}", + AUTH_METHOD="BASIC", + PROTOCOL="NATIVE", + LOGIN="{ch_user}", + PASSWORD_SECRET_NAME="ch_password_obj" + ); + )sql", + "ch_host"_a = GetChHost(), + "ch_port"_a = GetChPort(), + "ch_database"_a = GetChDatabase(), + "ch_user"_a = GetChUser(), + "ch_password"_a = GetChPassword()); + auto result = session.ExecuteSchemeQuery(sql).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + const TString sql = R"sql( + PRAGMA generic.UsePredicatePushdown="true"; + SELECT * FROM ch_data_source.ch_table_plan_test + WHERE name IS NOT NULL + )sql"; + + auto queryClient = kikimr->GetQueryClient(); + TExecuteQueryResult queryResult = queryClient.ExecuteQuery( + sql, + TTxControl::BeginTx().CommitTx(), + TExecuteQuerySettings().ExecMode(EExecMode::Explain)) + .GetValueSync(); + + UNIT_ASSERT_C(queryResult.IsSuccess(), queryResult.GetIssues().ToString()); + UNIT_ASSERT(queryResult.GetStats()); + UNIT_ASSERT(queryResult.GetStats()->GetPlan()); + Cerr << "Plan: " << *queryResult.GetStats()->GetPlan() << Endl; + NJson::TJsonValue plan; + UNIT_ASSERT(NJson::ReadJsonTree(*queryResult.GetStats()->GetPlan(), &plan)); + + const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Filter-Source"); + const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[1]; + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "ch_data_source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Database"].GetStringSafe(), GetChDatabase()); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Protocol"].GetStringSafe(), "Native"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Table"].GetStringSafe(), "ch_table_plan_test"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Read from external data source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["SourceType"].GetStringSafe(), "ClickHouse"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[1].GetStringSafe(), "name"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Filter"].GetStringSafe(), "Exist(item.name)"); + } +} diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp b/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp index a0fcae19479..c23005f3d0e 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp @@ -1,37 +1,16 @@ #include "ch_recipe_ut_helpers.h" #include "connector_recipe_ut_helpers.h" #include "pg_recipe_ut_helpers.h" -#include -#include -#include #include #include -#include - #include +using namespace NTestUtils; using namespace fmt::literals; -std::shared_ptr MakeKikimrRunner() { - NYql::TGenericConnectorConfig clientCfg; - clientCfg.MutableEndpoint()->set_host(GetConnectorHost()); - clientCfg.MutableEndpoint()->set_port(GetConnectorPort()); - - NKikimrConfig::TAppConfig appCfg; - appCfg.MutableFeatureFlags()->SetEnableExternalDataSources(true); - - auto kikimr = NKikimr::NKqp::NFederatedQueryTest::MakeKikimrRunner( - NYql::IHTTPGateway::Make(), - NYql::NConnector::MakeClientGRPC(clientCfg), - nullptr, - appCfg); - kikimr->GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); - return kikimr; -} - Y_UNIT_TEST_SUITE(FederatedQueryJoin) { Y_UNIT_TEST(InnerJoinChPg) { pqxx::connection pgConnection = CreatePostgresqlConnection(); @@ -83,7 +62,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) { chClient.Execute(insertData); } - std::shared_ptr kikimr = MakeKikimrRunner(); + std::shared_ptr kikimr = MakeKikimrRunnerWithConnector(); auto tableCLient = kikimr->GetTableClient(); auto session = tableCLient.CreateSession().GetValueSync().GetSession(); diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp b/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp new file mode 100644 index 00000000000..54a158c7d72 --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp @@ -0,0 +1,40 @@ +#include "pg_recipe_ut_helpers.h" + +#include +#include + +#include + +using namespace fmt::literals; + +namespace NTestUtils { + + TString GetPgHost() { + return GetEnv("POSTGRES_RECIPE_HOST", "localhost"); + } + + ui32 GetPgPort() { + const TString port = GetEnv("POSTGRES_RECIPE_PORT"); + UNIT_ASSERT_C(port, "No postgresql port specified"); + return FromString(port); + } + + TString GetPgUser() { + return GetEnv("POSTGRES_RECIPE_USER"); + } + + TString GetPgDatabase() { + return GetEnv("POSTGRES_RECIPE_DBNAME"); + } + + pqxx::connection CreatePostgresqlConnection() { + const TString connectionString = fmt::format( + "host={host} port={port} dbname={database} user={user}", + "host"_a = GetPgHost(), + "port"_a = GetPgPort(), + "database"_a = GetPgDatabase(), + "user"_a = GetPgUser()); + return pqxx::connection{connectionString}; + } + +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h b/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h index da00ec66055..c8c7bacffc3 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h @@ -1,36 +1,15 @@ #pragma once - -#include -#include - -#include +#include #include -using namespace fmt::literals; - -inline const TString GetPgHost() { - return GetEnv("POSTGRES_RECIPE_HOST", "localhost"); -} - -inline ui32 GetPgPort() { - return FromString(GetEnv("POSTGRES_RECIPE_PORT")); -} +namespace NTestUtils { -inline const TString GetPgUser() { - return GetEnv("POSTGRES_RECIPE_USER"); -} + TString GetPgHost(); + ui32 GetPgPort(); + TString GetPgUser(); + TString GetPgDatabase(); -inline const TString GetPgDatabase() { - return GetEnv("POSTGRES_RECIPE_DBNAME"); -} + pqxx::connection CreatePostgresqlConnection(); -inline pqxx::connection CreatePostgresqlConnection() { - const TString connectionString = fmt::format( - "host={host} port={port} dbname={database} user={user}", - "host"_a = GetPgHost(), - "port"_a = GetPgPort(), - "database"_a = GetPgDatabase(), - "user"_a = GetPgUser()); - return pqxx::connection{connectionString}; -} +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/generic/ya.make b/contrib/ydb/core/kqp/ut/federated_query/generic/ya.make index 8c21c9d601c..a30dabb360e 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/generic/ya.make +++ b/contrib/ydb/core/kqp/ut/federated_query/generic/ya.make @@ -3,7 +3,11 @@ UNITTEST_FOR(contrib/ydb/core/kqp) FORK_SUBTESTS() SRCS( + ch_recipe_ut_helpers.cpp + connector_recipe_ut_helpers.cpp + kqp_generic_plan_ut.cpp kqp_generic_provider_join_ut.cpp + pg_recipe_ut_helpers.cpp ) PEERDIR( diff --git a/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp b/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp index 82a053e053a..99f40e7bfe2 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp +++ b/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp @@ -1,12 +1,4 @@ -#include - -#include -#include -#include -#include -#include -#include -#include +#include "s3_recipe_ut_helpers.h" #include #include @@ -19,158 +11,16 @@ #include -#include - - -namespace NKikimr { -namespace NKqp { +namespace NKikimr::NKqp { using namespace NYdb; using namespace NYdb::NQuery; using namespace NKikimr::NKqp::NFederatedQueryTest; - -constexpr TStringBuf TEST_CONTENT = -R"({"key": "1", "value": "trololo"} -{"key": "2", "value": "hello world"} -)"sv; - -constexpr TStringBuf TEST_CONTENT_KEYS = -R"({"key": "1"} -{"key": "3"} -)"sv; - -const TString TEST_SCHEMA = R"(["StructType";[["key";["DataType";"Utf8";];];["value";["DataType";"Utf8";];];];])"; - -const TString TEST_SCHEMA_IDS = R"(["StructType";[["key";["DataType";"Utf8";];];];])"; - -Aws::S3::S3Client MakeS3Client() { - Aws::Client::ClientConfiguration s3ClientConfig; - s3ClientConfig.endpointOverride = GetEnv("S3_ENDPOINT"); - s3ClientConfig.scheme = Aws::Http::Scheme::HTTP; - return Aws::S3::S3Client( - std::make_shared(), - s3ClientConfig, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - /*useVirtualAddressing=*/ true); -} - -void CreateBucket(const TString& bucket, Aws::S3::S3Client& s3Client) { - Aws::S3::Model::CreateBucketRequest req; - req.SetBucket(bucket); - req.SetACL(Aws::S3::Model::BucketCannedACL::public_read_write); - const Aws::S3::Model::CreateBucketOutcome result = s3Client.CreateBucket(req); - UNIT_ASSERT_C(result.IsSuccess(), "Error creating bucket \"" << bucket << "\": " << result.GetError().GetExceptionName() << ": " << result.GetError().GetMessage()); -} - -void CreateBucket(const TString& bucket) { - Aws::S3::S3Client s3Client = MakeS3Client(); - - CreateBucket(bucket, s3Client); -} - -void UploadObject(const TString& bucket, const TString& object, const TStringBuf& content, Aws::S3::S3Client& s3Client) { - Aws::S3::Model::PutObjectRequest req; - req.WithBucket(bucket).WithKey(object); - - auto inputStream = std::make_shared(); - *inputStream << content; - req.SetBody(inputStream); - const Aws::S3::Model::PutObjectOutcome result = s3Client.PutObject(req); - UNIT_ASSERT_C(result.IsSuccess(), "Error uploading object \"" << object << "\" to a bucket \"" << bucket << "\": " << result.GetError().GetExceptionName() << ": " << result.GetError().GetMessage()); -} - -void UploadObject(const TString& bucket, const TString& object, const TStringBuf& content) { - Aws::S3::S3Client s3Client = MakeS3Client(); - - UploadObject(bucket, object, content, s3Client); -} - -void CreateBucketWithObject(const TString& bucket, const TString& object, const TStringBuf& content, Aws::S3::S3Client& s3Client) { - CreateBucket(bucket, s3Client); - UploadObject(bucket, object, content, s3Client); -} - -void CreateBucketWithObject(const TString& bucket, const TString& object, const TStringBuf& content) { - Aws::S3::S3Client s3Client = MakeS3Client(); - - CreateBucketWithObject(bucket, object, content, s3Client); -} - -TString GetObject(const TString& bucket, const TString& object, Aws::S3::S3Client& s3Client) { - Aws::S3::Model::GetObjectRequest req; - req.WithBucket(bucket).WithKey(object); - - Aws::S3::Model::GetObjectOutcome outcome = s3Client.GetObject(req); - UNIT_ASSERT(outcome.IsSuccess()); - Aws::S3::Model::GetObjectResult& result = outcome.GetResult(); - std::istreambuf_iterator eos; - std::string objContent(std::istreambuf_iterator(result.GetBody()), eos); - Cerr << "Got object content from \"" << bucket << "." << object << "\"\n" << objContent << Endl; - return objContent; -} - -TString GetObject(const TString& bucket, const TString& object) { - Aws::S3::S3Client s3Client = MakeS3Client(); - - return GetObject(bucket, object, s3Client); -} - -std::vector GetObjectKeys(const TString& bucket, Aws::S3::S3Client& s3Client) { - Aws::S3::Model::ListObjectsRequest listReq; - listReq.WithBucket(bucket); - - Aws::S3::Model::ListObjectsOutcome outcome = s3Client.ListObjects(listReq); - UNIT_ASSERT(outcome.IsSuccess()); - - std::vector keys; - for (auto& obj : outcome.GetResult().GetContents()) { - keys.push_back(TString(obj.GetKey())); - Cerr << "Found S3 object: \"" << obj.GetKey() << "\"" << Endl; - } - return keys; -} - -std::vector GetObjectKeys(const TString& bucket) { - Aws::S3::S3Client s3Client = MakeS3Client(); - - return GetObjectKeys(bucket, s3Client); -} - -TString GetAllObjects(const TString& bucket, TStringBuf separator, Aws::S3::S3Client& s3Client) { - std::vector keys = GetObjectKeys(bucket, s3Client); - TString result; - bool firstObject = true; - for (const TString& key : keys) { - result += GetObject(bucket, key, s3Client); - if (!firstObject) { - result += separator; - } - firstObject = false; - } - return result; -} - -TString GetAllObjects(const TString& bucket, TStringBuf separator = "") { - Aws::S3::S3Client s3Client = MakeS3Client(); - - return GetAllObjects(bucket, separator, s3Client); -} - -TString GetBucketLocation(const TStringBuf bucket) { - return TStringBuilder() << GetEnv("S3_ENDPOINT") << '/' << bucket << '/'; -} - -Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Aws::SDKOptions()); -} - -Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Aws::SDKOptions()); -} +using namespace NTestUtils; +using namespace fmt::literals; Y_UNIT_TEST_SUITE(KqpFederatedQuery) { Y_UNIT_TEST(ExecuteScriptWithExternalTableResolve) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket1"; @@ -233,7 +83,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteQueryWithExternalTableResolve) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket_execute_query"; @@ -305,8 +154,7 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } } - Y_UNIT_TEST(ExecuteScriptWithS3ReadNotCached) { - using namespace fmt::literals; + Y_UNIT_TEST(ExecuteScriptWithS3ReadNotCached) { const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket1"; @@ -362,7 +210,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithDataSource) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString bucket = "test_bucket3"; @@ -412,7 +259,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithDataSourceJoinYdb) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source_2"; const TString ydbTable = "/Root/ydb_table"; const TString bucket = "test_bucket4"; @@ -487,7 +333,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithExternalTableResolveCheckPragma) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket5"; @@ -550,7 +395,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithDataSourceJoinYdbCheckPragma) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source_2"; const TString ydbTable = "/Root/ydb_table"; const TString bucket = "test_bucket6"; @@ -628,7 +472,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithDataSourceAndTablePathPrefix) { - using namespace fmt::literals; const TString externalDataSourceName = "external_data_source"; const TString bucket = "test_bucket7"; @@ -678,7 +521,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } std::pair ExecuteScriptOverBinding(NKikimrConfig::TTableServiceConfig::EBindingsMode mode) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket1"; @@ -785,7 +627,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(InsertIntoBucket) { - using namespace fmt::literals; const TString readDataSourceName = "/Root/read_data_source"; const TString readTableName = "/Root/read_binding"; const TString readBucket = "test_bucket_read"; @@ -865,7 +706,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } void ExecuteInsertQuery(TQueryClient& client, const TString& writeTableName, const TString& readTableName, bool expectCached) { - using namespace fmt::literals; const TString sql = fmt::format(R"( INSERT INTO `{write_table}` SELECT * FROM `{read_table}`; @@ -881,7 +721,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(InsertIntoBucketCaching) { - using namespace fmt::literals; const TString writeDataSourceName = "/Root/write_data_source"; const TString writeTableName = "/Root/write_binding"; const TString writeBucket = "test_bucket_cache"; @@ -965,7 +804,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(UpdateExternalTable) { - using namespace fmt::literals; const TString readDataSourceName = "/Root/read_data_source"; const TString readTableName = "/Root/read_binding"; const TString readBucket = "test_bucket_read"; @@ -1013,7 +851,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(JoinTwoSources) { - using namespace fmt::literals; const TString dataSource = "/Root/data_source"; const TString bucket = "test_bucket_mixed"; const TString dataTable = "/Root/data"; @@ -1088,7 +925,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithExternalTableResolveCheckPartitionedBy) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket1"; @@ -1149,7 +985,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithEmptyCustomPartitioning) { - using namespace fmt::literals; const TString bucket = "test_bucket1"; const TString object = "year=2021/test_object"; @@ -1213,8 +1048,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithTruncatedMultiplyResults) { - using namespace fmt::literals; - const TString bucket = "test_bucket"; CreateBucket(bucket); @@ -1278,7 +1111,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ForbiddenCallablesForYdbTables) { - using namespace fmt::literals; const TString readDataSourceName = "/Root/read_data_source"; const TString readTableName = "/Root/read_table"; const TString readBucket = "test_read_bucket_forbidden_callables"; @@ -1385,7 +1217,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } Y_UNIT_TEST(ExecuteScriptWithLocationWithoutSlashAtTheEnd) { - using namespace fmt::literals; const TString externalDataSourceName = "/Root/external_data_source"; const TString externalTableName = "/Root/test_binding_resolve"; const TString bucket = "test_bucket_with_location_without_slash_at_the_end"; @@ -1446,5 +1277,4 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { } } -} // namespace NKqp -} // namespace NKikimr +} // namespace NKikimr::NKqp diff --git a/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp b/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp new file mode 100644 index 00000000000..7df17d668e6 --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp @@ -0,0 +1,163 @@ +#include "s3_recipe_ut_helpers.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NQuery; +using namespace NKikimr::NKqp::NFederatedQueryTest; +using namespace NTestUtils; +using namespace fmt::literals; + +Y_UNIT_TEST_SUITE(KqpS3PlanTest) { + Y_UNIT_TEST(S3Source) { + CreateBucketWithObject("test_bucket_plan_s3_source", "test_object_plan_s3_source", TEST_CONTENT); + + auto kikimr = MakeKikimrRunner(NYql::IHTTPGateway::Make()); + + auto tc = kikimr->GetTableClient(); + auto session = tc.CreateSession().GetValueSync().GetSession(); + const TString query = fmt::format(R"sql( + CREATE EXTERNAL DATA SOURCE external_data_source WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="{location}", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE external_table ( + key Utf8 NOT NULL, + value Utf8 NOT NULL + ) WITH ( + DATA_SOURCE="external_data_source", + LOCATION="test_object_plan_s3_source", + FORMAT="json_each_row" + );)sql", + "location"_a = GetBucketLocation("test_bucket_plan_s3_source") + ); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + + const TString sql = R"sql(SELECT * FROM external_table)sql"; + + auto queryClient = kikimr->GetQueryClient(); + TExecuteQueryResult queryResult = queryClient.ExecuteQuery( + sql, + TTxControl::BeginTx().CommitTx(), + TExecuteQuerySettings().ExecMode(EExecMode::Explain)).GetValueSync(); + + UNIT_ASSERT_C(queryResult.IsSuccess(), queryResult.GetIssues().ToString()); + UNIT_ASSERT(queryResult.GetStats()); + UNIT_ASSERT(queryResult.GetStats()->GetPlan()); + Cerr << "Plan: " << *queryResult.GetStats()->GetPlan() << Endl; + NJson::TJsonValue plan; + UNIT_ASSERT(NJson::ReadJsonTree(*queryResult.GetStats()->GetPlan(), &plan)); + + const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Stage-Source"); + const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[0]; + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "external_data_source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Format"].GetStringSafe(), "json_each_row"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Parse from external data source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["SourceType"].GetStringSafe(), "s3"); + UNIT_ASSERT(!IsIn(sourceOp.GetMap(), "RowsLimitHint")); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[1].GetStringSafe(), "value"); + } + + Y_UNIT_TEST(S3Sink) { + { + Aws::S3::S3Client s3Client = MakeS3Client(); + CreateBucketWithObject("test_bucket_read", "test_object_read", TEST_CONTENT, s3Client); + CreateBucket("test_bucket_write", s3Client); + } + + auto kikimr = MakeKikimrRunner(NYql::IHTTPGateway::Make()); + + auto tc = kikimr->GetTableClient(); + auto session = tc.CreateSession().GetValueSync().GetSession(); + const TString query = fmt::format(R"sql( + CREATE EXTERNAL DATA SOURCE read_data_source WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="{read_location}", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE read_table ( + key Utf8 NOT NULL, + value Utf8 NOT NULL + ) WITH ( + DATA_SOURCE="read_data_source", + LOCATION="test_object_read", + FORMAT="json_each_row" + ); + + CREATE EXTERNAL DATA SOURCE write_data_source WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="{write_location}", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE write_table ( + key Utf8 NOT NULL, + value Utf8 NOT NULL + ) WITH ( + DATA_SOURCE="write_data_source", + LOCATION="test_object_write/", + FORMAT="json_each_row", + COMPRESSION="gzip" + ); + )sql", + "read_location"_a = GetBucketLocation("test_bucket_read"), + "write_location"_a = GetBucketLocation("test_bucket_write") + ); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + + const TString sql = R"sql( + INSERT INTO write_table + SELECT * FROM read_table + LIMIT 10 + )sql"; + + auto queryClient = kikimr->GetQueryClient(); + TExecuteQueryResult queryResult = queryClient.ExecuteQuery( + sql, + TTxControl::BeginTx().CommitTx(), + TExecuteQuerySettings().ExecMode(EExecMode::Explain)).GetValueSync(); + + UNIT_ASSERT_C(queryResult.IsSuccess(), queryResult.GetIssues().ToString()); + UNIT_ASSERT(queryResult.GetStats()); + UNIT_ASSERT(queryResult.GetStats()->GetPlan()); + Cerr << "Plan: " << *queryResult.GetStats()->GetPlan() << Endl; + NJson::TJsonValue plan; + UNIT_ASSERT(NJson::ReadJsonTree(*queryResult.GetStats()->GetPlan(), &plan)); + + const auto& writeStagePlan = plan["Plan"]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(writeStagePlan["Node Type"].GetStringSafe(), "Limit-Sink"); + UNIT_ASSERT(writeStagePlan["Operators"].GetArraySafe().size() >= 2); + const auto& sinkOp = writeStagePlan["Operators"].GetArraySafe()[1]; + UNIT_ASSERT_VALUES_EQUAL(sinkOp["ExternalDataSource"].GetStringSafe(), "write_data_source"); + UNIT_ASSERT_VALUES_EQUAL(sinkOp["Compression"].GetStringSafe(), "gzip"); + + const auto& readStagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(readStagePlan["Node Type"].GetStringSafe(), "Limit-Source"); + const auto& sourceOp = readStagePlan["Operators"].GetArraySafe()[1]; + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "read_data_source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["RowsLimitHint"].GetStringSafe(), "10"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[1].GetStringSafe(), "value"); + } +} + +} // namespace NKikimr::NKqp diff --git a/contrib/ydb/core/kqp/ut/federated_query/s3/s3_recipe_ut_helpers.cpp b/contrib/ydb/core/kqp/ut/federated_query/s3/s3_recipe_ut_helpers.cpp new file mode 100644 index 00000000000..edb534201e6 --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/s3/s3_recipe_ut_helpers.cpp @@ -0,0 +1,141 @@ +#include "s3_recipe_ut_helpers.h" + +#include + +#include + +#include + +Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { + Aws::InitAPI(Aws::SDKOptions()); +} + +Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { + Aws::ShutdownAPI(Aws::SDKOptions()); +} + +namespace NTestUtils { + + extern const TString TEST_SCHEMA = R"(["StructType";[["key";["DataType";"Utf8";];];["value";["DataType";"Utf8";];];];])"; + + extern const TString TEST_SCHEMA_IDS = R"(["StructType";[["key";["DataType";"Utf8";];];];])"; + + Aws::S3::S3Client MakeS3Client() { + Aws::Client::ClientConfiguration s3ClientConfig; + s3ClientConfig.endpointOverride = GetEnv("S3_ENDPOINT"); + s3ClientConfig.scheme = Aws::Http::Scheme::HTTP; + return Aws::S3::S3Client( + std::make_shared(), + s3ClientConfig, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + /*useVirtualAddressing=*/true); + } + + void CreateBucket(const TString& bucket, Aws::S3::S3Client& s3Client) { + Aws::S3::Model::CreateBucketRequest req; + req.SetBucket(bucket); + req.SetACL(Aws::S3::Model::BucketCannedACL::public_read_write); + const Aws::S3::Model::CreateBucketOutcome result = s3Client.CreateBucket(req); + UNIT_ASSERT_C(result.IsSuccess(), "Error creating bucket \"" << bucket << "\": " << result.GetError().GetExceptionName() << ": " << result.GetError().GetMessage()); + } + + void CreateBucket(const TString& bucket) { + Aws::S3::S3Client s3Client = MakeS3Client(); + + CreateBucket(bucket, s3Client); + } + + void UploadObject(const TString& bucket, const TString& object, const TStringBuf& content, Aws::S3::S3Client& s3Client) { + Aws::S3::Model::PutObjectRequest req; + req.WithBucket(bucket).WithKey(object); + + auto inputStream = std::make_shared(); + *inputStream << content; + req.SetBody(inputStream); + const Aws::S3::Model::PutObjectOutcome result = s3Client.PutObject(req); + UNIT_ASSERT_C(result.IsSuccess(), "Error uploading object \"" << object << "\" to a bucket \"" << bucket << "\": " << result.GetError().GetExceptionName() << ": " << result.GetError().GetMessage()); + } + + void UploadObject(const TString& bucket, const TString& object, const TStringBuf& content) { + Aws::S3::S3Client s3Client = MakeS3Client(); + + UploadObject(bucket, object, content, s3Client); + } + + void CreateBucketWithObject(const TString& bucket, const TString& object, const TStringBuf& content, Aws::S3::S3Client& s3Client) { + CreateBucket(bucket, s3Client); + UploadObject(bucket, object, content, s3Client); + } + + void CreateBucketWithObject(const TString& bucket, const TString& object, const TStringBuf& content) { + Aws::S3::S3Client s3Client = MakeS3Client(); + + CreateBucketWithObject(bucket, object, content, s3Client); + } + + TString GetObject(const TString& bucket, const TString& object, Aws::S3::S3Client& s3Client) { + Aws::S3::Model::GetObjectRequest req; + req.WithBucket(bucket).WithKey(object); + + Aws::S3::Model::GetObjectOutcome outcome = s3Client.GetObject(req); + UNIT_ASSERT(outcome.IsSuccess()); + Aws::S3::Model::GetObjectResult& result = outcome.GetResult(); + std::istreambuf_iterator eos; + std::string objContent(std::istreambuf_iterator(result.GetBody()), eos); + Cerr << "Got object content from \"" << bucket << "." << object << "\"\n" + << objContent << Endl; + return objContent; + } + + TString GetObject(const TString& bucket, const TString& object) { + Aws::S3::S3Client s3Client = MakeS3Client(); + + return GetObject(bucket, object, s3Client); + } + + std::vector GetObjectKeys(const TString& bucket, Aws::S3::S3Client& s3Client) { + Aws::S3::Model::ListObjectsRequest listReq; + listReq.WithBucket(bucket); + + Aws::S3::Model::ListObjectsOutcome outcome = s3Client.ListObjects(listReq); + UNIT_ASSERT(outcome.IsSuccess()); + + std::vector keys; + for (auto& obj : outcome.GetResult().GetContents()) { + keys.push_back(TString(obj.GetKey())); + Cerr << "Found S3 object: \"" << obj.GetKey() << "\"" << Endl; + } + return keys; + } + + std::vector GetObjectKeys(const TString& bucket) { + Aws::S3::S3Client s3Client = MakeS3Client(); + + return GetObjectKeys(bucket, s3Client); + } + + TString GetAllObjects(const TString& bucket, TStringBuf separator, Aws::S3::S3Client& s3Client) { + std::vector keys = GetObjectKeys(bucket, s3Client); + TString result; + bool firstObject = true; + for (const TString& key : keys) { + result += GetObject(bucket, key, s3Client); + if (!firstObject) { + result += separator; + } + firstObject = false; + } + return result; + } + + TString GetAllObjects(const TString& bucket, TStringBuf separator) { + Aws::S3::S3Client s3Client = MakeS3Client(); + + return GetAllObjects(bucket, separator, s3Client); + } + + TString GetBucketLocation(const TStringBuf bucket) { + return TStringBuilder() << GetEnv("S3_ENDPOINT") << '/' << bucket << '/'; + } + +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/s3/s3_recipe_ut_helpers.h b/contrib/ydb/core/kqp/ut/federated_query/s3/s3_recipe_ut_helpers.h new file mode 100644 index 00000000000..d20024c3cea --- /dev/null +++ b/contrib/ydb/core/kqp/ut/federated_query/s3/s3_recipe_ut_helpers.h @@ -0,0 +1,50 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace NTestUtils { + + constexpr TStringBuf TEST_CONTENT = + R"({"key": "1", "value": "trololo"} + {"key": "2", "value": "hello world"})"sv; + + constexpr TStringBuf TEST_CONTENT_KEYS = + R"({"key": "1"} + {"key": "3"})"sv; + + extern const TString TEST_SCHEMA; + extern const TString TEST_SCHEMA_IDS; + + Aws::S3::S3Client MakeS3Client(); + + void CreateBucket(const TString& bucket, Aws::S3::S3Client& s3Client); + void CreateBucket(const TString& bucket); + + void UploadObject(const TString& bucket, const TString& object, const TStringBuf& content, Aws::S3::S3Client& s3Client); + void UploadObject(const TString& bucket, const TString& object, const TStringBuf& content); + + void CreateBucketWithObject(const TString& bucket, const TString& object, const TStringBuf& content, Aws::S3::S3Client& s3Client); + void CreateBucketWithObject(const TString& bucket, const TString& object, const TStringBuf& content); + + TString GetObject(const TString& bucket, const TString& object, Aws::S3::S3Client& s3Client); + TString GetObject(const TString& bucket, const TString& object); + + std::vector GetObjectKeys(const TString& bucket, Aws::S3::S3Client& s3Client); + std::vector GetObjectKeys(const TString& bucket); + + TString GetAllObjects(const TString& bucket, TStringBuf separator, Aws::S3::S3Client& s3Client); + TString GetAllObjects(const TString& bucket, TStringBuf separator = {}); + + TString GetBucketLocation(const TStringBuf bucket); + +} // namespace NTestUtils diff --git a/contrib/ydb/core/kqp/ut/federated_query/s3/ya.make b/contrib/ydb/core/kqp/ut/federated_query/s3/ya.make index b69bf06ddf2..b03a576def7 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/s3/ya.make +++ b/contrib/ydb/core/kqp/ut/federated_query/s3/ya.make @@ -11,6 +11,8 @@ ENDIF() SRCS( kqp_federated_query_ut.cpp + kqp_s3_plan_ut.cpp + s3_recipe_ut_helpers.cpp ) PEERDIR( diff --git a/contrib/ydb/core/kqp/ut/federated_query/style/ya.make b/contrib/ydb/core/kqp/ut/federated_query/style/ya.make index 22cc591a4c8..0fee747c56b 100644 --- a/contrib/ydb/core/kqp/ut/federated_query/style/ya.make +++ b/contrib/ydb/core/kqp/ut/federated_query/style/ya.make @@ -11,5 +11,3 @@ STYLE( ) END() - - diff --git a/contrib/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/contrib/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index eb2437a0428..ebf4b35ddc9 100644 --- a/contrib/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/contrib/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -248,10 +248,10 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { auto indexTableAccess = CountPlanNodesByKv(plan, "Table", "tg/tg_index/indexImplTable"); UNIT_ASSERT_VALUES_EQUAL(indexTableAccess, 1); - auto filterOnIndex = CountPlanNodesByKv(plan, "Node Type", "Limit-Filter-TablePointLookup"); + auto filterOnIndex = CountPlanNodesByKv(plan, "Node Type", "Limit-Filter-TableRangeScan"); UNIT_ASSERT_VALUES_EQUAL(filterOnIndex, 1); - auto limitFilterNode = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TablePointLookup"); + auto limitFilterNode = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TableRangeScan"); auto val = FindPlanNodes(limitFilterNode, "Limit"); UNIT_ASSERT_VALUES_EQUAL(val.size(), 1); UNIT_ASSERT_VALUES_EQUAL(val[0], "11"); diff --git a/contrib/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp b/contrib/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp index 2806f458e0f..a989fe6ebb8 100644 --- a/contrib/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp +++ b/contrib/ydb/core/kqp/ut/opt/kqp_extract_predicate_unpack_ut.cpp @@ -106,7 +106,7 @@ void PrepareTablesToUnpack(TSession session) { Y_UNIT_TEST_SUITE(KqpExtractPredicateLookup) { -void Test(const TString& query, const TString& answer, TMaybe allowScans = {}, NYdb::TParams params = TParamsBuilder().Build()) { +void Test(const TString& query, const TString& answer, THashSet allowScans = {}, NYdb::TParams params = TParamsBuilder().Build()) { TKikimrSettings settings; settings.SetDomainRoot(KikimrDefaultUtDomainRoot); TKikimrRunner kikimr(settings); @@ -133,7 +133,7 @@ void Test(const TString& query, const TString& answer, TMaybe allowScan UNIT_ASSERT(ValidatePlanNodeIds(plan)); for (const auto& tableStats : plan.GetMap().at("tables").GetArray()) { TString table = tableStats.GetMap().at("name").GetString(); - if (allowScans && table == *allowScans) { + if (allowScans.contains(table)) { continue; } @@ -406,7 +406,7 @@ Y_UNIT_TEST(PointJoin) { [[2];[102];["Value1"];[102];["Value22"]]; [[2];[103];["Value3"];[103];["Value23"]] ])", - "/Root/SimpleKey", + {"/Root/SimpleKey"}, TParamsBuilder().AddParam("$p").Int32(1).Build().Build()); Test( @@ -421,7 +421,7 @@ Y_UNIT_TEST(PointJoin) { R"([ [[3u];[103];["Value2"];[103];["Value23"]] ])", - "/Root/SimpleKey", + {"/Root/SimpleKey"}, TParamsBuilder().AddParam("$p").Int32(3).Build().Build()); Test( @@ -435,7 +435,7 @@ Y_UNIT_TEST(PointJoin) { )", R"([ ])", - "/Root/SimpleKey", + {"/Root/SimpleKey"}, TParamsBuilder().AddParam("$p").Int32(-2).Build().Build()); Test( @@ -450,7 +450,7 @@ Y_UNIT_TEST(PointJoin) { R"([ [[3u];[103];["Value2"];[103];["Value23"];["103-2"]] ])", - "/Root/SimpleKey", + {"/Root/SimpleKey", "/Root/UintComplexKeyWithIndex/Index/indexImplTable"}, TParamsBuilder().AddParam("$p").Int32(3).Build().Build()); } @@ -467,7 +467,7 @@ Y_UNIT_TEST(SqlInJoin) { [[2];[102];["Value1"]]; [[2];[103];["Value3"]] ])", - "/Root/SimpleKey", + {"/Root/SimpleKey"}, TParamsBuilder().AddParam("$p").Int32(1).Build().Build()); Test( @@ -480,7 +480,7 @@ Y_UNIT_TEST(SqlInJoin) { R"([ [[3u];[103];["Value2"]] ])", - "/Root/SimpleKey", + {"/Root/SimpleKey"}, TParamsBuilder().AddParam("$p").Int32(3).Build().Build()); Test( @@ -492,7 +492,7 @@ Y_UNIT_TEST(SqlInJoin) { )", R"([ ])", - "/Root/SimpleKey", + {"/Root/SimpleKey"}, TParamsBuilder().AddParam("$p").Int32(-2).Build().Build()); @@ -506,7 +506,7 @@ Y_UNIT_TEST(SqlInJoin) { R"([ [[3u];[103];["Value2"];["103-2"]] ])", - "/Root/SimpleKey", + {"/Root/SimpleKey", "/Root/UintComplexKeyWithIndex/Index/indexImplTable"}, TParamsBuilder().AddParam("$p").Int32(3).Build().Build()); } diff --git a/contrib/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp b/contrib/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp index 22af9f64c9b..90a22aaa58f 100644 --- a/contrib/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp +++ b/contrib/ydb/core/kqp/ut/opt/kqp_sort_ut.cpp @@ -1054,21 +1054,21 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto tableLookup = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TablePointLookup"); + auto tableRangeRead = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TableRangeScan"); size_t lookupIndex = 2; - if (!tableLookup.IsDefined()) { - tableLookup = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); + if (!tableRangeRead.IsDefined()) { + tableRangeRead = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); lookupIndex = 1; } - UNIT_ASSERT(tableLookup.IsDefined()); + UNIT_ASSERT(tableRangeRead.IsDefined()); - auto& limitOp = tableLookup.GetMapSafe().at("Operators").GetArraySafe().at(0).GetMapSafe(); + auto& limitOp = tableRangeRead.GetMapSafe().at("Operators").GetArraySafe().at(0).GetMapSafe(); UNIT_ASSERT_VALUES_EQUAL("Limit", limitOp.at("Name").GetStringSafe()); UNIT_ASSERT_VALUES_EQUAL("$limit", limitOp.at("Limit").GetStringSafe()); - auto& lookupOp = tableLookup.GetMapSafe().at("Operators").GetArraySafe().at(lookupIndex).GetMapSafe(); - UNIT_ASSERT_VALUES_EQUAL("TablePointLookup", lookupOp.at("Name").GetStringSafe()); - UNIT_ASSERT_VALUES_EQUAL("index", lookupOp.at("Table").GetStringSafe()); + auto& rangeReadOp = tableRangeRead.GetMapSafe().at("Operators").GetArraySafe().at(lookupIndex).GetMapSafe(); + UNIT_ASSERT_VALUES_EQUAL("TableRangeScan", rangeReadOp.at("Name").GetStringSafe()); + UNIT_ASSERT_VALUES_EQUAL("index", rangeReadOp.at("Table").GetStringSafe()); } Y_UNIT_TEST(Offset) { diff --git a/contrib/ydb/core/kqp/ut/query/kqp_explain_ut.cpp b/contrib/ydb/core/kqp/ut/query/kqp_explain_ut.cpp index 79465f88b25..cce37c55af5 100644 --- a/contrib/ydb/core/kqp/ut/query/kqp_explain_ut.cpp +++ b/contrib/ydb/core/kqp/ut/query/kqp_explain_ut.cpp @@ -277,20 +277,20 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::ReadJsonTree(*res.PlanJson, &plan, true); UNIT_ASSERT(ValidatePlanNodeIds(plan)); - auto read = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); + auto read = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); size_t operatorsCount = 2; size_t lookupMember = 1; if (!read.IsDefined()) { - read = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TablePointLookup"); + read = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TableRangeScan"); operatorsCount = 3; lookupMember = 2; } auto& operators = read.GetMapSafe().at("Operators").GetArraySafe(); UNIT_ASSERT(operators.size() == operatorsCount); - auto& lookup = operators[lookupMember].GetMapSafe(); - UNIT_ASSERT(lookup.at("Name") == "TablePointLookup"); - UNIT_ASSERT_VALUES_EQUAL(lookup.at("ReadRange").GetArraySafe()[0], "App («new_app_1»)"); + auto& rangeRead = operators[lookupMember].GetMapSafe(); + UNIT_ASSERT(rangeRead.at("Name") == "TableRangeScan"); + UNIT_ASSERT_VALUES_EQUAL(rangeRead.at("ReadRange").GetArraySafe()[0], "App («new_app_1»)"); } Y_UNIT_TEST(SortStage) { @@ -703,9 +703,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::ReadJsonTree(*res.PlanJson, &plan, true); UNIT_ASSERT(ValidatePlanNodeIds(plan)); - auto read = FindPlanNodeByKv(plan, "Node Type", "TableRangesScan"); + auto read = FindPlanNodeByKv(plan, "Node Type", "TableRangeScan"); if (!read.IsDefined()) { - read = FindPlanNodeByKv(plan, "Name", "TableRangesScan"); + read = FindPlanNodeByKv(plan, "Name", "TableRangeScan"); } UNIT_ASSERT(read.IsDefined()); auto keys = FindPlanNodeByKv(plan, "ReadRangesKeys", "[\"Key\"]"); diff --git a/contrib/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp b/contrib/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp index 80d50a460f4..08c8eaa1593 100644 --- a/contrib/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp +++ b/contrib/ydb/core/kqp/ut/scan/kqp_scan_ut.cpp @@ -2474,9 +2474,9 @@ Y_UNIT_TEST_SUITE(KqpScan) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); + auto indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); if (!indexRead.IsDefined()) { - indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TablePointLookup"); + indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TableRangeScan"); } UNIT_ASSERT(indexRead.IsDefined()); auto indexTable = FindPlanNodeByKv(indexRead, "Table", "SecondaryComplexKeys/Index/indexImplTable"); @@ -2514,9 +2514,9 @@ Y_UNIT_TEST_SUITE(KqpScan) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); + auto indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); if (!indexRead.IsDefined()) { - indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TablePointLookup"); + indexRead = FindPlanNodeByKv(plan, "Node Type", "Limit-Filter-TableRangeScan"); } UNIT_ASSERT(indexRead.IsDefined()); auto indexTable = FindPlanNodeByKv(indexRead, "Table", "SecondaryComplexKeys/Index/indexImplTable"); diff --git a/contrib/ydb/core/kqp/ut/service/kqp_service_ut.cpp b/contrib/ydb/core/kqp/ut/service/kqp_service_ut.cpp index d7d6adae34f..cd61bcac290 100644 --- a/contrib/ydb/core/kqp/ut/service/kqp_service_ut.cpp +++ b/contrib/ydb/core/kqp/ut/service/kqp_service_ut.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -95,10 +96,24 @@ Y_UNIT_TEST_SUITE(KqpService) { } Cerr << "finished sessions close....." << Endl; - auto counters = kikimr->GetTestServer().GetRuntime()->GetAppData(0).Counters; + auto counters = GetServiceCounters(kikimr->GetTestServer().GetRuntime()->GetAppData(0).Counters, "ydb"); + + ui64 pendingCompilations = 0; + do { + Sleep(WaitDuration); + pendingCompilations = counters->GetNamedCounter("name", "table.query.compilation.active_count", false)->Val(); + Cerr << "still compiling... " << pendingCompilations << Endl; + } while (pendingCompilations != 0); + + ui64 pendingSessions = 0; do { Sleep(WaitDuration); - } while (counters->GetNamedCounter("name", "table.query.compilation.active_count", false)->Val() != 0); + pendingSessions = counters->GetNamedCounter("name", "table.session.active_count", false)->Val(); + Cerr << "still active sessions ... " << pendingCompilations << Endl; + } while (pendingSessions != 0); + + Sleep(TDuration::Seconds(5)); + return; } diff --git a/contrib/ydb/core/protos/cms.proto b/contrib/ydb/core/protos/cms.proto index 39a37abba16..4bb330e6624 100644 --- a/contrib/ydb/core/protos/cms.proto +++ b/contrib/ydb/core/protos/cms.proto @@ -161,6 +161,8 @@ message TPermissionRequest { // Availability mode is not preserved for scheduled events. optional EAvailabilityMode AvailabilityMode = 9 [default = MODE_MAX_AVAILABILITY]; optional string MaintenanceTaskId = 10; + // Prepare item (host/device) before granting permission + optional bool Prepare = 11 [default = false]; } enum EExtensionType { diff --git a/contrib/ydb/core/protos/config.proto b/contrib/ydb/core/protos/config.proto index 73f832d5bc7..b2a92a7174f 100644 --- a/contrib/ydb/core/protos/config.proto +++ b/contrib/ydb/core/protos/config.proto @@ -752,6 +752,25 @@ message THttpProxyConfig { optional string JwtFile = 10; } +message TS3ProxyResolverConfig { + message THttpResolverConfig { + // resolve proxy host to connect to that endpoint using this url + optional string ResolveUrl = 1; + // use resolved proxy through these ports + optional uint32 HttpPort = 2; + optional uint32 HttpsPort = 3; + } + + message TEndpoint { + // S3 endpoint + optional string Endpoint = 1; + oneof Resolver { + THttpResolverConfig HttpResolver = 2; + } + } + + repeated TEndpoint Endpoints = 1; +} message TSqsConfig { optional bool EnableSqs = 5; @@ -1671,6 +1690,7 @@ message TAppConfig { optional TQueryServiceConfig QueryServiceConfig = 73; optional TConveyorConfig InsertConveyorConfig = 74; optional bool AllowEditYamlInUi = 75; + optional TS3ProxyResolverConfig S3ProxyResolverConfig = 76; repeated TNamedConfig NamedConfigs = 100; optional string ClusterYamlConfig = 101; diff --git a/contrib/ydb/core/protos/console_config.proto b/contrib/ydb/core/protos/console_config.proto index dd03829b91d..e7694fa3dea 100644 --- a/contrib/ydb/core/protos/console_config.proto +++ b/contrib/ydb/core/protos/console_config.proto @@ -135,6 +135,7 @@ message TConfigItem { QueryServiceConfigItem = 73; InsertConveyorConfigItem = 74; AllowEditYamlInUiItem = 75; + S3ProxyResolverConfigItem = 76; NamedConfigsItem = 100; ClusterYamlConfigItem = 101; diff --git a/contrib/ydb/core/protos/out/out_cms.cpp b/contrib/ydb/core/protos/out/out_cms.cpp index a3685e7de4d..3a15708c066 100644 --- a/contrib/ydb/core/protos/out/out_cms.cpp +++ b/contrib/ydb/core/protos/out/out_cms.cpp @@ -25,3 +25,7 @@ Y_DECLARE_OUT_SPEC(, NKikimrCms::ETextFormat, stream, value) { Y_DECLARE_OUT_SPEC(, NKikimrCms::TLogRecordData::EType, stream, value) { stream << NKikimrCms::TLogRecordData::EType_Name(value); } + +Y_DECLARE_OUT_SPEC(, NKikimrCms::TAction::EType, stream, value) { + stream << NKikimrCms::TAction::EType_Name(value); +} diff --git a/contrib/ydb/core/testlib/actors/test_runtime.cpp b/contrib/ydb/core/testlib/actors/test_runtime.cpp index 5260e4027f7..8f046af006c 100644 --- a/contrib/ydb/core/testlib/actors/test_runtime.cpp +++ b/contrib/ydb/core/testlib/actors/test_runtime.cpp @@ -150,6 +150,7 @@ namespace NActors { nodeAppData->ColumnShardConfig = app0->ColumnShardConfig; nodeAppData->MeteringConfig = app0->MeteringConfig; nodeAppData->AwsCompatibilityConfig = app0->AwsCompatibilityConfig; + nodeAppData->S3ProxyResolverConfig = app0->S3ProxyResolverConfig; nodeAppData->EnableMvccSnapshotWithLegacyDomainRoot = app0->EnableMvccSnapshotWithLegacyDomainRoot; nodeAppData->IoContextFactory = app0->IoContextFactory; if (KeyConfigGenerator) { diff --git a/contrib/ydb/core/testlib/basics/appdata.cpp b/contrib/ydb/core/testlib/basics/appdata.cpp index b089f8e9891..ae9a4fed0d1 100644 --- a/contrib/ydb/core/testlib/basics/appdata.cpp +++ b/contrib/ydb/core/testlib/basics/appdata.cpp @@ -57,6 +57,7 @@ namespace NKikimr { app->SchemeShardConfig = SchemeShardConfig; app->MeteringConfig = MeteringConfig; app->AwsCompatibilityConfig = AwsCompatibilityConfig; + app->S3ProxyResolverConfig = S3ProxyResolverConfig; app->FeatureFlags = FeatureFlags; // This is a special setting active in test runtime only diff --git a/contrib/ydb/core/testlib/basics/appdata.h b/contrib/ydb/core/testlib/basics/appdata.h index 1cad512caba..18959ffc25a 100644 --- a/contrib/ydb/core/testlib/basics/appdata.h +++ b/contrib/ydb/core/testlib/basics/appdata.h @@ -96,6 +96,7 @@ namespace NKikimr { NKikimrConfig::TMeteringConfig MeteringConfig; NKikimrPQ::TPQConfig PQConfig; NKikimrConfig::TAwsCompatibilityConfig AwsCompatibilityConfig; + NKikimrConfig::TS3ProxyResolverConfig S3ProxyResolverConfig; private: TAutoPtr Mine; diff --git a/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp b/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp index 76a01747599..b9d4f1d4a95 100644 --- a/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp +++ b/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp @@ -1,6 +1,7 @@ #include "abstract_scheme.h" #include +#include namespace NKikimr::NOlap { @@ -103,4 +104,10 @@ std::shared_ptr ISnapshotSchema::PrepareForInsert(const TStr return batch; } +ui32 ISnapshotSchema::GetColumnId(const std::string& columnName) const { + auto id = GetColumnIdOptional(columnName); + AFL_VERIFY(id)("column_name", columnName)("schema", JoinSeq(",", GetSchema()->field_names())); + return *id; +} + } diff --git a/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h b/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h index b09e3daac6a..4e6c16c5683 100644 --- a/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h +++ b/contrib/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h @@ -45,11 +45,7 @@ class ISnapshotSchema { virtual std::optional GetColumnIdOptional(const std::string& columnName) const = 0; virtual int GetFieldIndex(const ui32 columnId) const = 0; - ui32 GetColumnId(const std::string& columnName) const { - auto id = GetColumnIdOptional(columnName); - AFL_VERIFY(id); - return *id; - } + ui32 GetColumnId(const std::string& columnName) const; std::shared_ptr GetFieldByIndex(const int index) const; std::shared_ptr GetFieldByColumnId(const ui32 columnId) const; std::shared_ptr GetFieldByColumnIdVerified(const ui32 columnId) const { diff --git a/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 4715f9acf57..b318a5b25b0 100644 --- a/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -127,27 +127,26 @@ std::vector TIndexInfo::GetColumns(const std::vector& ids) return NOlap::GetColumns(*this, ids); } -std::shared_ptr TIndexInfo::ArrowSchema() const { - if (!Schema) { - std::vector ids; - ids.reserve(Columns.size()); - for (const auto& [id, _] : Columns) { - ids.push_back(id); - } - - // The ids had a set type before so we keep them sorted. - std::sort(ids.begin(), ids.end()); - Schema = MakeArrowSchema(Columns, ids); +void TIndexInfo::BuildArrowSchema() { + AFL_VERIFY(!Schema); + std::vector ids; + ids.reserve(Columns.size()); + for (const auto& [id, _] : Columns) { + ids.push_back(id); } - return Schema; + // The ids had a set type before so we keep them sorted. + std::sort(ids.begin(), ids.end()); + Schema = MakeArrowSchema(Columns, ids); } -std::shared_ptr TIndexInfo::ArrowSchemaWithSpecials() const { - if (SchemaWithSpecials) { - return SchemaWithSpecials; - } +std::shared_ptr TIndexInfo::ArrowSchema() const { + AFL_VERIFY(Schema); + return Schema; +} +void TIndexInfo::BuildSchemaWithSpecials() { + AFL_VERIFY(!SchemaWithSpecials); const auto& schema = ArrowSchema(); std::vector> extended; @@ -160,6 +159,10 @@ std::shared_ptr TIndexInfo::ArrowSchemaWithSpecials() const { extended.insert(extended.end(), schema->fields().begin(), schema->fields().end()); SchemaWithSpecials = std::make_shared(std::move(extended)); +} + +std::shared_ptr TIndexInfo::ArrowSchemaWithSpecials() const { + AFL_VERIFY(SchemaWithSpecials); return SchemaWithSpecials; } @@ -244,6 +247,11 @@ void TIndexInfo::SetAllKeys() { } } MinMaxIdxColumnsIds.insert(GetPKFirstColumnId()); + if (!Schema) { + AFL_VERIFY(!SchemaWithSpecials); + BuildArrowSchema(); + BuildSchemaWithSpecials(); + } } std::shared_ptr TIndexInfo::SortDescription() const { @@ -360,6 +368,8 @@ bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& Columns[id] = NTable::TColumn(name, id, typeInfoMod.TypeInfo, typeInfoMod.TypeMod, notNull); ColumnNames[name] = id; } + BuildArrowSchema(); + BuildSchemaWithSpecials(); for (const auto& col : schema.GetColumns()) { std::optional cFeatures = TColumnFeatures::BuildFromProto(col, *this); if (!cFeatures) { @@ -382,7 +392,6 @@ bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& } DefaultCompression = *result; } - Version = schema.GetVersion(); return true; } diff --git a/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.h b/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.h index fd124c660e6..0dff0b00773 100644 --- a/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/contrib/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -37,6 +37,9 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { TIndexInfo(const TString& name, ui32 id); bool DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema); TColumnFeatures& GetOrCreateColumnFeatures(const ui32 columnId) const; + void BuildSchemaWithSpecials(); + void BuildArrowSchema(); + public: static constexpr const char* SPEC_COL_PLAN_STEP = "_yql_plan_step"; static constexpr const char* SPEC_COL_TX_ID = "_yql_tx_id"; @@ -207,8 +210,8 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { ui32 Id; ui64 Version = 0; TString Name; - mutable std::shared_ptr Schema; - mutable std::shared_ptr SchemaWithSpecials; + std::shared_ptr Schema; + std::shared_ptr SchemaWithSpecials; std::shared_ptr SortingKey; std::shared_ptr ReplaceKey; std::shared_ptr ExtendedKey; // Extend PK with snapshot columns to allow old shapshot reads diff --git a/contrib/ydb/core/tx/datashard/CMakeLists.darwin-x86_64.txt b/contrib/ydb/core/tx/datashard/CMakeLists.darwin-x86_64.txt index 2a0e76d49b1..2628cec1178 100644 --- a/contrib/ydb/core/tx/datashard/CMakeLists.darwin-x86_64.txt +++ b/contrib/ydb/core/tx/datashard/CMakeLists.darwin-x86_64.txt @@ -78,6 +78,7 @@ target_link_libraries(core-tx-datashard PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 @@ -346,6 +347,7 @@ target_link_libraries(core-tx-datashard.global PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 diff --git a/contrib/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt b/contrib/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt index 72a3b1b947b..1cecdb2c2db 100644 --- a/contrib/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt +++ b/contrib/ydb/core/tx/datashard/CMakeLists.linux-aarch64.txt @@ -79,6 +79,7 @@ target_link_libraries(core-tx-datashard PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 @@ -348,6 +349,7 @@ target_link_libraries(core-tx-datashard.global PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 diff --git a/contrib/ydb/core/tx/datashard/CMakeLists.linux-x86_64.txt b/contrib/ydb/core/tx/datashard/CMakeLists.linux-x86_64.txt index 72a3b1b947b..1cecdb2c2db 100644 --- a/contrib/ydb/core/tx/datashard/CMakeLists.linux-x86_64.txt +++ b/contrib/ydb/core/tx/datashard/CMakeLists.linux-x86_64.txt @@ -79,6 +79,7 @@ target_link_libraries(core-tx-datashard PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 @@ -348,6 +349,7 @@ target_link_libraries(core-tx-datashard.global PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 diff --git a/contrib/ydb/core/tx/datashard/CMakeLists.windows-x86_64.txt b/contrib/ydb/core/tx/datashard/CMakeLists.windows-x86_64.txt index 7d4b3e543e5..3711c40fd8f 100644 --- a/contrib/ydb/core/tx/datashard/CMakeLists.windows-x86_64.txt +++ b/contrib/ydb/core/tx/datashard/CMakeLists.windows-x86_64.txt @@ -79,6 +79,7 @@ target_link_libraries(core-tx-datashard PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 @@ -343,6 +344,7 @@ target_link_libraries(core-tx-datashard.global PUBLIC library-cpp-resource contrib-libs-zstd library-actors-core + library-actors-http cpp-containers-absl_flat_hash cpp-containers-stack_vector cpp-digest-md5 diff --git a/contrib/ydb/core/tx/datashard/export_s3_base_uploader.h b/contrib/ydb/core/tx/datashard/export_s3_base_uploader.h deleted file mode 100644 index e7f492e9e37..00000000000 --- a/contrib/ydb/core/tx/datashard/export_s3_base_uploader.h +++ /dev/null @@ -1,548 +0,0 @@ -#pragma once -#ifndef KIKIMR_DISABLE_S3_OPS - -#include "datashard.h" -#include "export_common.h" -#include "export_s3.h" -#include "extstorage_usage_config.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -namespace NKikimr { -namespace NDataShard { - -class IProxyOps { -public: - virtual ~IProxyOps() = default; - virtual bool NeedToResolveProxy() const = 0; - virtual void ResolveProxy() = 0; - -}; // IProxyOps - -template -class TS3UploaderBase: public TActorBootstrapped - , public IProxyOps -{ - using TEvExternalStorage = NWrappers::TEvExternalStorage; - using TEvBuffer = TEvExportScan::TEvBuffer; - -protected: - void Restart() { - Y_ABORT_UNLESS(ProxyResolved); - - MultiPart = false; - Last = false; - Parts.clear(); - - if (Attempt) { - this->Send(std::exchange(Client, TActorId()), new TEvents::TEvPoisonPill()); - } - - Client = this->RegisterWithSameMailbox(NWrappers::CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); - - if (!MetadataUploaded) { - UploadMetadata(); - } else if (!SchemeUploaded) { - UploadScheme(); - } else { - this->Become(&TDerived::StateUploadData); - - if (Attempt) { - this->Send(std::exchange(Scanner, TActorId()), new TEvExportScan::TEvReset()); - } else if (Scanner) { - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - } - } - - void UploadScheme() { - Y_ABORT_UNLESS(!SchemeUploaded); - - if (!Scheme) { - return Finish(false, "Cannot infer scheme"); - } - - google::protobuf::TextFormat::PrintToString(Scheme.GetRef(), &Buffer); - - auto request = Aws::S3::Model::PutObjectRequest() - .WithKey(Settings.GetSchemeKey()) - .WithStorageClass(Settings.GetStorageClass()); - this->Send(Client, new TEvExternalStorage::TEvPutObjectRequest(request, std::move(Buffer))); - - this->Become(&TDerived::StateUploadScheme); - } - - void UploadMetadata() { - Y_ABORT_UNLESS(!MetadataUploaded); - - Buffer = std::move(Metadata); - - auto request = Aws::S3::Model::PutObjectRequest() - .WithKey(Settings.GetMetadataKey()) - .WithStorageClass(Settings.GetStorageClass()); - this->Send(Client, new TEvExternalStorage::TEvPutObjectRequest(request, std::move(Buffer))); - - this->Become(&TDerived::StateUploadMetadata); - } - - void HandleScheme(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("HandleScheme TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!CheckResult(result, TStringBuf("PutObject (scheme)"))) { - return; - } - - SchemeUploaded = true; - - if (Scanner) { - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - - this->Become(&TDerived::StateUploadData); - } - - void HandleMetadata(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("HandleMetadata TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!CheckResult(result, TStringBuf("PutObject (metadata)"))) { - return; - } - - MetadataUploaded = true; - - UploadScheme(); - } - - void Handle(TEvExportScan::TEvReady::TPtr& ev) { - EXPORT_LOG_D("Handle TEvExportScan::TEvReady" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender); - - Scanner = ev->Sender; - - if (Error) { - return PassAway(); - } - - if (ProxyResolved && SchemeUploaded && MetadataUploaded) { - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - } - - void Handle(TEvBuffer::TPtr& ev) { - EXPORT_LOG_D("Handle TEvExportScan::TEvBuffer" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender - << ", msg# " << ev->Get()->ToString()); - - if (ev->Sender != Scanner) { - EXPORT_LOG_W("Received buffer from unknown scanner" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender - << ", scanner# " << Scanner); - return; - } - - Last = ev->Get()->Last; - MultiPart = MultiPart || !Last; - ev->Get()->Buffer.AsString(Buffer); - - UploadData(); - } - - void UploadData() { - if (!MultiPart) { - auto request = Aws::S3::Model::PutObjectRequest() - .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) - .WithStorageClass(Settings.GetStorageClass()); - this->Send(Client, new TEvExternalStorage::TEvPutObjectRequest(request, std::move(Buffer))); - } else { - if (!UploadId) { - this->Send(DataShard, new TEvDataShard::TEvGetS3Upload(this->SelfId(), TxId)); - return; - } - - auto request = Aws::S3::Model::UploadPartRequest() - .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) - .WithUploadId(*UploadId) - .WithPartNumber(Parts.size() + 1); - this->Send(Client, new TEvExternalStorage::TEvUploadPartRequest(request, std::move(Buffer))); - } - } - - void HandleData(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("HandleData TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!CheckResult(result, TStringBuf("PutObject (data)"))) { - return; - } - - Finish(); - } - - void Handle(TEvDataShard::TEvS3Upload::TPtr& ev) { - auto& upload = ev->Get()->Upload; - - EXPORT_LOG_D("Handle TEvDataShard::TEvS3Upload" - << ": self# " << this->SelfId() - << ", upload# " << upload); - - if (!upload) { - auto request = Aws::S3::Model::CreateMultipartUploadRequest() - .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) - .WithStorageClass(Settings.GetStorageClass()); - this->Send(Client, new TEvExternalStorage::TEvCreateMultipartUploadRequest(request)); - } else { - UploadId = upload->Id; - - switch (upload->Status) { - case TS3Upload::EStatus::UploadParts: - return UploadData(); - - case TS3Upload::EStatus::Complete: { - Parts = std::move(upload->Parts); - - TVector parts(Reserve(Parts.size())); - for (ui32 partIndex = 0; partIndex < Parts.size(); ++partIndex) { - parts.emplace_back(Aws::S3::Model::CompletedPart() - .WithPartNumber(partIndex + 1) - .WithETag(Parts.at(partIndex))); - } - - auto request = Aws::S3::Model::CompleteMultipartUploadRequest() - .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) - .WithUploadId(*UploadId) - .WithMultipartUpload(Aws::S3::Model::CompletedMultipartUpload().WithParts(std::move(parts))); - this->Send(Client, new TEvExternalStorage::TEvCompleteMultipartUploadRequest(request)); - break; - } - - case TS3Upload::EStatus::Abort: { - Error = std::move(upload->Error); - if (!Error) { - Error = ""; - } - - auto request = Aws::S3::Model::AbortMultipartUploadRequest() - .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) - .WithUploadId(*UploadId); - this->Send(Client, new TEvExternalStorage::TEvAbortMultipartUploadRequest(request)); - break; - } - } - } - } - - void Handle(TEvExternalStorage::TEvCreateMultipartUploadResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvExternalStorage::TEvCreateMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!CheckResult(result, TStringBuf("CreateMultipartUpload"))) { - return; - } - - this->Send(DataShard, new TEvDataShard::TEvStoreS3UploadId(this->SelfId(), TxId, result.GetResult().GetUploadId().c_str())); - } - - void Handle(TEvExternalStorage::TEvUploadPartResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvExternalStorage::TEvUploadPartResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (!CheckResult(result, TStringBuf("UploadPart"))) { - return; - } - - Parts.push_back(result.GetResult().GetETag().c_str()); - - if (Last) { - return Finish(); - } - - this->Send(Scanner, new TEvExportScan::TEvFeed()); - } - - void Handle(TEvExternalStorage::TEvCompleteMultipartUploadResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvExternalStorage::TEvCompleteMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (result.IsSuccess()) { - return PassAway(); - } - - const auto& error = result.GetError(); - if (error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_UPLOAD) { - return PassAway(); - } - - if (CanRetry(error)) { - UploadId.Clear(); // force getting info after restart - Retry(); - } else { - Error = error.GetMessage().c_str(); - PassAway(); - } - } - - void Handle(TEvExternalStorage::TEvAbortMultipartUploadResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - EXPORT_LOG_D("Handle TEvExternalStorage::TEvAbortMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); - - if (result.IsSuccess()) { - return PassAway(); - } - - const auto& error = result.GetError(); - if (CanRetry(error)) { - UploadId.Clear(); // force getting info after restart - Retry(); - } else { - Y_ABORT_UNLESS(Error); - Error = TStringBuilder() << *Error << " Additionally, 'AbortMultipartUpload' has failed: " - << error.GetMessage(); - PassAway(); - } - } - - template - bool CheckResult(const TResult& result, const TStringBuf marker) { - if (result.IsSuccess()) { - return true; - } - - EXPORT_LOG_E("Error at '" << marker << "'" - << ": self# " << this->SelfId() - << ", error# " << result); - RetryOrFinish(result.GetError()); - - return false; - } - - static bool ShouldRetry(const Aws::S3::S3Error& error) { - if (error.ShouldRetry()) { - return true; - } - - if ("TooManyRequests" == error.GetExceptionName()) { - return true; - } - - return false; - } - - bool CanRetry(const Aws::S3::S3Error& error) const { - return Attempt < Retries && ShouldRetry(error); - } - - void Retry() { - Delay = Min(Delay * ++Attempt, TDuration::Minutes(10)); - const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); - this->Schedule(Delay + random, new TEvents::TEvWakeup()); - } - - void RetryOrFinish(const Aws::S3::S3Error& error) { - if (CanRetry(error)) { - Retry(); - } else { - Finish(false, TStringBuilder() << "S3 error: " << error.GetMessage().c_str()); - } - } - - void Finish(bool success = true, const TString& error = TString()) { - EXPORT_LOG_I("Finish" - << ": self# " << this->SelfId() - << ", success# " << success - << ", error# " << error - << ", multipart# " << MultiPart - << ", uploadId# " << UploadId); - - if (!success) { - Error = error; - } - - if (!MultiPart || !UploadId) { - if (!Scanner) { - return; - } - - PassAway(); - } else { - if (success) { - this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, - TS3Upload::EStatus::Complete, std::move(Parts))); - } else { - this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, - TS3Upload::EStatus::Abort, *Error)); - } - } - } - - void PassAway() override { - if (Scanner) { - this->Send(Scanner, new TEvExportScan::TEvFinish(Error.Empty(), Error.GetOrElse(TString()))); - } - - this->Send(Client, new TEvents::TEvPoisonPill()); - - IActor::PassAway(); - } - -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::EXPORT_S3_UPLOADER_ACTOR; - } - - static constexpr TStringBuf LogPrefix() { - return "s3"sv; - } - - explicit TS3UploaderBase( - const TActorId& dataShard, ui64 txId, - const NKikimrSchemeOp::TBackupTask& task, - TMaybe&& scheme, - TString&& metadata) - : ExternalStorageConfig(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(task.GetS3Settings())) - , Settings(TS3Settings::FromBackupTask(task)) - , DataFormat(NBackupRestoreTraits::EDataFormat::Csv) - , CompressionCodec(NBackupRestoreTraits::CodecFromTask(task)) - , DataShard(dataShard) - , TxId(txId) - , Scheme(std::move(scheme)) - , Metadata(std::move(metadata)) - , Retries(task.GetNumberOfRetries()) - , Attempt(0) - , Delay(TDuration::Minutes(1)) - , SchemeUploaded(task.GetShardNum() == 0 ? false : true) - , MetadataUploaded(task.GetShardNum() == 0 ? false : true) - { - } - - void Bootstrap() { - EXPORT_LOG_D("Bootstrap" - << ": self# " << this->SelfId() - << ", attempt# " << Attempt); - - ProxyResolved = !NeedToResolveProxy(); - if (!ProxyResolved) { - ResolveProxy(); - } else { - Restart(); - } - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExportScan::TEvReady, Handle); - - sFunc(TEvents::TEvWakeup, Bootstrap); - sFunc(TEvents::TEvPoisonPill, PassAway); - } - } - - STATEFN(StateUploadScheme) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleScheme); - default: - return StateBase(ev); - } - } - - STATEFN(StateUploadMetadata) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleMetadata); - default: - return StateBase(ev); - } - } - - STATEFN(StateUploadData) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvBuffer, Handle); - hFunc(TEvDataShard::TEvS3Upload, Handle); - - hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleData); - hFunc(TEvExternalStorage::TEvCreateMultipartUploadResponse, Handle); - hFunc(TEvExternalStorage::TEvUploadPartResponse, Handle); - hFunc(TEvExternalStorage::TEvCompleteMultipartUploadResponse, Handle); - hFunc(TEvExternalStorage::TEvAbortMultipartUploadResponse, Handle); - default: - return StateBase(ev); - } - } - -protected: - NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig; - TS3Settings Settings; - const NBackupRestoreTraits::EDataFormat DataFormat; - const NBackupRestoreTraits::ECompressionCodec CompressionCodec; - bool ProxyResolved; - -private: - const TActorId DataShard; - const ui64 TxId; - const TMaybe Scheme; - const TString Metadata; - - const ui32 Retries; - ui32 Attempt; - - TActorId Client; - TDuration Delay; - bool SchemeUploaded; - bool MetadataUploaded; - bool MultiPart; - bool Last; - - TActorId Scanner; - TString Buffer; - - TMaybe UploadId; - TVector Parts; - TMaybe Error; - -}; // TS3UploaderBase - -} // NDataShard -} // NKikimr - -#endif // KIKIMR_DISABLE_S3_OPS diff --git a/contrib/ydb/core/tx/datashard/export_s3_uploader.cpp b/contrib/ydb/core/tx/datashard/export_s3_uploader.cpp index 0ff72f3f043..b1f8a9a6788 100644 --- a/contrib/ydb/core/tx/datashard/export_s3_uploader.cpp +++ b/contrib/ydb/core/tx/datashard/export_s3_uploader.cpp @@ -1,24 +1,655 @@ #ifndef KIKIMR_DISABLE_S3_OPS -#include "export_s3_base_uploader.h" - #include "backup_restore_common.h" +#include "datashard.h" +#include "export_common.h" +#include "export_s3.h" +#include "extstorage_usage_config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include namespace NKikimr { namespace NDataShard { -class TS3Uploader: public TS3UploaderBase { -protected: - bool NeedToResolveProxy() const override { +class TS3Uploader: public TActorBootstrapped { + using TS3ExternalStorageConfig = NWrappers::NExternalStorage::TS3ExternalStorageConfig; + using THttpResolverConfig = NKikimrConfig::TS3ProxyResolverConfig::THttpResolverConfig; + using TEvExternalStorage = NWrappers::TEvExternalStorage; + using TEvBuffer = TEvExportScan::TEvBuffer; + + static TMaybe GetHttpResolverConfig(TStringBuf endpoint) { + for (const auto& entry : AppData()->S3ProxyResolverConfig.GetEndpoints()) { + if (entry.GetEndpoint() == endpoint && entry.HasHttpResolver()) { + return entry.GetHttpResolver(); + } + } + + return Nothing(); + } + + static TStringBuf NormalizeEndpoint(TStringBuf endpoint) { + Y_UNUSED(endpoint.SkipPrefix("http://") || endpoint.SkipPrefix("https://")); + Y_UNUSED(endpoint.ChopSuffix(":80") || endpoint.ChopSuffix(":443")); + return endpoint; + } + + static TMaybe GetHttpResolverConfig(const TS3ExternalStorageConfig& settings) { + return GetHttpResolverConfig(NormalizeEndpoint(settings.GetConfig().endpointOverride)); + } + + std::shared_ptr GetS3StorageConfig() const { + return std::dynamic_pointer_cast(ExternalStorageConfig); + } + + TString GetResolveProxyUrl(const TS3ExternalStorageConfig& settings) const { + Y_ABORT_UNLESS(HttpResolverConfig); + + TStringBuilder url; + switch (settings.GetConfig().scheme) { + case Aws::Http::Scheme::HTTP: + url << "http://"; + break; + case Aws::Http::Scheme::HTTPS: + url << "https://"; + break; + } + + url << HttpResolverConfig->GetResolveUrl(); + return url; + } + + void ApplyProxy(TS3ExternalStorageConfig& settings, const TString& proxyHost) const { + Y_ABORT_UNLESS(HttpResolverConfig); + + settings.ConfigRef().proxyScheme = settings.GetConfig().scheme; + settings.ConfigRef().proxyHost = proxyHost; + settings.ConfigRef().proxyCaPath = settings.GetConfig().caPath; + + switch (settings.GetConfig().proxyScheme) { + case Aws::Http::Scheme::HTTP: + settings.ConfigRef().proxyPort = HttpResolverConfig->GetHttpPort(); + break; + case Aws::Http::Scheme::HTTPS: + settings.ConfigRef().proxyPort = HttpResolverConfig->GetHttpsPort(); + break; + } + } + + void ResolveProxy() { + if (!HttpProxy) { + HttpProxy = Register(NHttp::CreateHttpProxy(NMonitoring::TMetricRegistry::SharedInstance())); + } + + Send(HttpProxy, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest( + NHttp::THttpOutgoingRequest::CreateRequestGet(GetResolveProxyUrl(*GetS3StorageConfig())), + TDuration::Seconds(10) + )); + + Become(&TThis::StateResolveProxy); + } + + void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) { + const auto& msg = *ev->Get(); + + EXPORT_LOG_D("Handle NHttp::TEvHttpProxy::TEvHttpIncomingResponse" + << ": self# " << SelfId() + << ", status# " << (msg.Response ? msg.Response->Status : "null") + << ", body# " << (msg.Response ? msg.Response->Body : "null")); + + if (!msg.Response || !msg.Response->Status.StartsWith("200")) { + EXPORT_LOG_E("Error at 'GetProxy'" + << ": self# " << SelfId() + << ", error# " << msg.GetError()); + return RetryOrFinish(Aws::S3::S3Error({Aws::S3::S3Errors::SERVICE_UNAVAILABLE, true})); + } + + if (msg.Response->Body.find('<') != TStringBuf::npos) { + EXPORT_LOG_E("Error at 'GetProxy'" + << ": self# " << SelfId() + << ", error# " << "invalid body" + << ", body# " << msg.Response->Body); + return RetryOrFinish(Aws::S3::S3Error({Aws::S3::S3Errors::SERVICE_UNAVAILABLE, true})); + } + + ApplyProxy(*GetS3StorageConfig(), TString(msg.Response->Body)); + ProxyResolved = true; + + const auto& cfg = GetS3StorageConfig()->GetConfig(); + EXPORT_LOG_N("Using proxy: " + << (cfg.proxyScheme == Aws::Http::Scheme::HTTPS ? "https://" : "http://") + << cfg.proxyHost << ":" << cfg.proxyPort); + + Restart(); + } + + void Restart() { + Y_ABORT_UNLESS(ProxyResolved); + + MultiPart = false; + Last = false; + Parts.clear(); + + if (Attempt) { + this->Send(std::exchange(Client, TActorId()), new TEvents::TEvPoisonPill()); + } + + Client = this->RegisterWithSameMailbox(NWrappers::CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); + + if (!MetadataUploaded) { + UploadMetadata(); + } else if (!SchemeUploaded) { + UploadScheme(); + } else { + this->Become(&TThis::StateUploadData); + + if (Attempt) { + this->Send(std::exchange(Scanner, TActorId()), new TEvExportScan::TEvReset()); + } else if (Scanner) { + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + } + } + + void UploadScheme() { + Y_ABORT_UNLESS(!SchemeUploaded); + + if (!Scheme) { + return Finish(false, "Cannot infer scheme"); + } + + google::protobuf::TextFormat::PrintToString(Scheme.GetRef(), &Buffer); + + auto request = Aws::S3::Model::PutObjectRequest() + .WithKey(Settings.GetSchemeKey()) + .WithStorageClass(Settings.GetStorageClass()); + this->Send(Client, new TEvExternalStorage::TEvPutObjectRequest(request, std::move(Buffer))); + + this->Become(&TThis::StateUploadScheme); + } + + void UploadMetadata() { + Y_ABORT_UNLESS(!MetadataUploaded); + + Buffer = std::move(Metadata); + + auto request = Aws::S3::Model::PutObjectRequest() + .WithKey(Settings.GetMetadataKey()) + .WithStorageClass(Settings.GetStorageClass()); + this->Send(Client, new TEvExternalStorage::TEvPutObjectRequest(request, std::move(Buffer))); + + this->Become(&TThis::StateUploadMetadata); + } + + void HandleScheme(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("HandleScheme TEvExternalStorage::TEvPutObjectResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!CheckResult(result, TStringBuf("PutObject (scheme)"))) { + return; + } + + SchemeUploaded = true; + + if (Scanner) { + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + + this->Become(&TThis::StateUploadData); + } + + void HandleMetadata(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("HandleMetadata TEvExternalStorage::TEvPutObjectResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!CheckResult(result, TStringBuf("PutObject (metadata)"))) { + return; + } + + MetadataUploaded = true; + + UploadScheme(); + } + + void Handle(TEvExportScan::TEvReady::TPtr& ev) { + EXPORT_LOG_D("Handle TEvExportScan::TEvReady" + << ": self# " << this->SelfId() + << ", sender# " << ev->Sender); + + Scanner = ev->Sender; + + if (Error) { + return PassAway(); + } + + if (ProxyResolved && SchemeUploaded && MetadataUploaded) { + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + } + + void Handle(TEvBuffer::TPtr& ev) { + EXPORT_LOG_D("Handle TEvExportScan::TEvBuffer" + << ": self# " << this->SelfId() + << ", sender# " << ev->Sender + << ", msg# " << ev->Get()->ToString()); + + if (ev->Sender != Scanner) { + EXPORT_LOG_W("Received buffer from unknown scanner" + << ": self# " << this->SelfId() + << ", sender# " << ev->Sender + << ", scanner# " << Scanner); + return; + } + + Last = ev->Get()->Last; + MultiPart = MultiPart || !Last; + ev->Get()->Buffer.AsString(Buffer); + + UploadData(); + } + + void UploadData() { + if (!MultiPart) { + auto request = Aws::S3::Model::PutObjectRequest() + .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) + .WithStorageClass(Settings.GetStorageClass()); + this->Send(Client, new TEvExternalStorage::TEvPutObjectRequest(request, std::move(Buffer))); + } else { + if (!UploadId) { + this->Send(DataShard, new TEvDataShard::TEvGetS3Upload(this->SelfId(), TxId)); + return; + } + + auto request = Aws::S3::Model::UploadPartRequest() + .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) + .WithUploadId(*UploadId) + .WithPartNumber(Parts.size() + 1); + this->Send(Client, new TEvExternalStorage::TEvUploadPartRequest(request, std::move(Buffer))); + } + } + + void HandleData(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("HandleData TEvExternalStorage::TEvPutObjectResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!CheckResult(result, TStringBuf("PutObject (data)"))) { + return; + } + + Finish(); + } + + void Handle(TEvDataShard::TEvS3Upload::TPtr& ev) { + auto& upload = ev->Get()->Upload; + + EXPORT_LOG_D("Handle TEvDataShard::TEvS3Upload" + << ": self# " << this->SelfId() + << ", upload# " << upload); + + if (!upload) { + auto request = Aws::S3::Model::CreateMultipartUploadRequest() + .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) + .WithStorageClass(Settings.GetStorageClass()); + this->Send(Client, new TEvExternalStorage::TEvCreateMultipartUploadRequest(request)); + } else { + UploadId = upload->Id; + + switch (upload->Status) { + case TS3Upload::EStatus::UploadParts: + return UploadData(); + + case TS3Upload::EStatus::Complete: { + Parts = std::move(upload->Parts); + + TVector parts(Reserve(Parts.size())); + for (ui32 partIndex = 0; partIndex < Parts.size(); ++partIndex) { + parts.emplace_back(Aws::S3::Model::CompletedPart() + .WithPartNumber(partIndex + 1) + .WithETag(Parts.at(partIndex))); + } + + auto request = Aws::S3::Model::CompleteMultipartUploadRequest() + .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) + .WithUploadId(*UploadId) + .WithMultipartUpload(Aws::S3::Model::CompletedMultipartUpload().WithParts(std::move(parts))); + this->Send(Client, new TEvExternalStorage::TEvCompleteMultipartUploadRequest(request)); + break; + } + + case TS3Upload::EStatus::Abort: { + Error = std::move(upload->Error); + if (!Error) { + Error = ""; + } + + auto request = Aws::S3::Model::AbortMultipartUploadRequest() + .WithKey(Settings.GetDataKey(DataFormat, CompressionCodec)) + .WithUploadId(*UploadId); + this->Send(Client, new TEvExternalStorage::TEvAbortMultipartUploadRequest(request)); + break; + } + } + } + } + + void Handle(TEvExternalStorage::TEvCreateMultipartUploadResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvExternalStorage::TEvCreateMultipartUploadResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!CheckResult(result, TStringBuf("CreateMultipartUpload"))) { + return; + } + + this->Send(DataShard, new TEvDataShard::TEvStoreS3UploadId(this->SelfId(), TxId, result.GetResult().GetUploadId().c_str())); + } + + void Handle(TEvExternalStorage::TEvUploadPartResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvExternalStorage::TEvUploadPartResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (!CheckResult(result, TStringBuf("UploadPart"))) { + return; + } + + Parts.push_back(result.GetResult().GetETag().c_str()); + + if (Last) { + return Finish(); + } + + this->Send(Scanner, new TEvExportScan::TEvFeed()); + } + + void Handle(TEvExternalStorage::TEvCompleteMultipartUploadResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvExternalStorage::TEvCompleteMultipartUploadResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (result.IsSuccess()) { + return PassAway(); + } + + const auto& error = result.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_UPLOAD) { + return PassAway(); + } + + if (CanRetry(error)) { + UploadId.Clear(); // force getting info after restart + Retry(); + } else { + Error = error.GetMessage().c_str(); + PassAway(); + } + } + + void Handle(TEvExternalStorage::TEvAbortMultipartUploadResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + EXPORT_LOG_D("Handle TEvExternalStorage::TEvAbortMultipartUploadResponse" + << ": self# " << this->SelfId() + << ", result# " << result); + + if (result.IsSuccess()) { + return PassAway(); + } + + const auto& error = result.GetError(); + if (CanRetry(error)) { + UploadId.Clear(); // force getting info after restart + Retry(); + } else { + Y_ABORT_UNLESS(Error); + Error = TStringBuilder() << *Error << " Additionally, 'AbortMultipartUpload' has failed: " + << error.GetMessage(); + PassAway(); + } + } + + template + bool CheckResult(const TResult& result, const TStringBuf marker) { + if (result.IsSuccess()) { + return true; + } + + EXPORT_LOG_E("Error at '" << marker << "'" + << ": self# " << this->SelfId() + << ", error# " << result); + RetryOrFinish(result.GetError()); + + return false; + } + + static bool ShouldRetry(const Aws::S3::S3Error& error) { + if (error.ShouldRetry()) { + return true; + } + + if ("TooManyRequests" == error.GetExceptionName()) { + return true; + } + return false; } - void ResolveProxy() override { - Y_ABORT("unreachable"); + bool CanRetry(const Aws::S3::S3Error& error) const { + return Attempt < Retries && ShouldRetry(error); + } + + void Retry() { + Delay = Min(Delay * ++Attempt, TDuration::Minutes(10)); + const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); + this->Schedule(Delay + random, new TEvents::TEvWakeup()); + } + + void RetryOrFinish(const Aws::S3::S3Error& error) { + if (CanRetry(error)) { + Retry(); + } else { + Finish(false, TStringBuilder() << "S3 error: " << error.GetMessage().c_str()); + } + } + + void Finish(bool success = true, const TString& error = TString()) { + EXPORT_LOG_I("Finish" + << ": self# " << this->SelfId() + << ", success# " << success + << ", error# " << error + << ", multipart# " << MultiPart + << ", uploadId# " << UploadId); + + if (!success) { + Error = error; + } + + if (!MultiPart || !UploadId) { + if (!Scanner) { + return; + } + + PassAway(); + } else { + if (success) { + this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, + TS3Upload::EStatus::Complete, std::move(Parts))); + } else { + this->Send(DataShard, new TEvDataShard::TEvChangeS3UploadStatus(this->SelfId(), TxId, + TS3Upload::EStatus::Abort, *Error)); + } + } + } + + void PassAway() override { + if (HttpProxy) { + Send(HttpProxy, new TEvents::TEvPoisonPill()); + } + + if (Scanner) { + this->Send(Scanner, new TEvExportScan::TEvFinish(Error.Empty(), Error.GetOrElse(TString()))); + } + + this->Send(Client, new TEvents::TEvPoisonPill()); + + IActor::PassAway(); } public: - using TS3UploaderBase::TS3UploaderBase; + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::EXPORT_S3_UPLOADER_ACTOR; + } + + static constexpr TStringBuf LogPrefix() { + return "s3"sv; + } + + explicit TS3Uploader( + const TActorId& dataShard, ui64 txId, + const NKikimrSchemeOp::TBackupTask& task, + TMaybe&& scheme, + TString&& metadata) + : ExternalStorageConfig(new TS3ExternalStorageConfig(task.GetS3Settings())) + , Settings(TS3Settings::FromBackupTask(task)) + , DataFormat(NBackupRestoreTraits::EDataFormat::Csv) + , CompressionCodec(NBackupRestoreTraits::CodecFromTask(task)) + , HttpResolverConfig(GetHttpResolverConfig(*GetS3StorageConfig())) + , DataShard(dataShard) + , TxId(txId) + , Scheme(std::move(scheme)) + , Metadata(std::move(metadata)) + , Retries(task.GetNumberOfRetries()) + , Attempt(0) + , Delay(TDuration::Minutes(1)) + , SchemeUploaded(task.GetShardNum() == 0 ? false : true) + , MetadataUploaded(task.GetShardNum() == 0 ? false : true) + { + } + + void Bootstrap() { + EXPORT_LOG_D("Bootstrap" + << ": self# " << this->SelfId() + << ", attempt# " << Attempt); + + ProxyResolved = !HttpResolverConfig.Defined(); + if (!ProxyResolved) { + ResolveProxy(); + } else { + Restart(); + } + } + + STATEFN(StateBase) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExportScan::TEvReady, Handle); + + sFunc(TEvents::TEvWakeup, Bootstrap); + sFunc(TEvents::TEvPoisonPill, PassAway); + } + } + + STATEFN(StateResolveProxy) { + switch (ev->GetTypeRewrite()) { + hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle); + default: + return StateBase(ev); + } + } + + STATEFN(StateUploadScheme) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleScheme); + default: + return StateBase(ev); + } + } + + STATEFN(StateUploadMetadata) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleMetadata); + default: + return StateBase(ev); + } + } + + STATEFN(StateUploadData) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvBuffer, Handle); + hFunc(TEvDataShard::TEvS3Upload, Handle); + + hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleData); + hFunc(TEvExternalStorage::TEvCreateMultipartUploadResponse, Handle); + hFunc(TEvExternalStorage::TEvUploadPartResponse, Handle); + hFunc(TEvExternalStorage::TEvCompleteMultipartUploadResponse, Handle); + hFunc(TEvExternalStorage::TEvAbortMultipartUploadResponse, Handle); + default: + return StateBase(ev); + } + } + +private: + NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig; + TS3Settings Settings; + const NBackupRestoreTraits::EDataFormat DataFormat; + const NBackupRestoreTraits::ECompressionCodec CompressionCodec; + bool ProxyResolved; + + TMaybe HttpResolverConfig; + TActorId HttpProxy; + + const TActorId DataShard; + const ui64 TxId; + const TMaybe Scheme; + const TString Metadata; + + const ui32 Retries; + ui32 Attempt; + + TActorId Client; + TDuration Delay; + bool SchemeUploaded; + bool MetadataUploaded; + bool MultiPart; + bool Last; + + TActorId Scanner; + TString Buffer; + + TMaybe UploadId; + TVector Parts; + TMaybe Error; }; // TS3Uploader diff --git a/contrib/ydb/core/tx/datashard/ya.make b/contrib/ydb/core/tx/datashard/ya.make index fe2c5ebbde2..b6aab34728d 100644 --- a/contrib/ydb/core/tx/datashard/ya.make +++ b/contrib/ydb/core/tx/datashard/ya.make @@ -216,6 +216,7 @@ RESOURCE( PEERDIR( contrib/libs/zstd contrib/ydb/library/actors/core + contrib/ydb/library/actors/http library/cpp/containers/absl_flat_hash library/cpp/containers/stack_vector library/cpp/digest/md5 diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp b/contrib/ydb/library/fyamlcpp/fyamlcpp_ut.cpp similarity index 100% rename from library/cpp/yaml/fyamlcpp/fyamlcpp_ut.cpp rename to contrib/ydb/library/fyamlcpp/fyamlcpp_ut.cpp diff --git a/library/cpp/yaml/fyamlcpp/libfyaml_ut.cpp b/contrib/ydb/library/fyamlcpp/libfyaml_ut.cpp similarity index 100% rename from library/cpp/yaml/fyamlcpp/libfyaml_ut.cpp rename to contrib/ydb/library/fyamlcpp/libfyaml_ut.cpp diff --git a/contrib/ydb/library/fyamlcpp/ut/ya.make b/contrib/ydb/library/fyamlcpp/ut/ya.make index 1ac2b254388..0935f080b40 100644 --- a/contrib/ydb/library/fyamlcpp/ut/ya.make +++ b/contrib/ydb/library/fyamlcpp/ut/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(library/cpp/yaml/fyamlcpp) +UNITTEST_FOR(contrib/ydb/library/fyamlcpp) SRCS( fyamlcpp_ut.cpp diff --git a/contrib/ydb/library/yql/core/services/yql_lineage.cpp b/contrib/ydb/library/yql/core/services/yql_lineage.cpp index 608d555b6eb..5c2973be4e7 100644 --- a/contrib/ydb/library/yql/core/services/yql_lineage.cpp +++ b/contrib/ydb/library/yql/core/services/yql_lineage.cpp @@ -41,7 +41,8 @@ class TLineageScanner { writer.OnBeginList(); for (const auto& r : Reads_) { TVector inputs; - r.second->GetPlanFormatter().GetInputs(*r.first, inputs); + auto& formatter = r.second->GetPlanFormatter(); + formatter.GetInputs(*r.first, inputs); for (const auto& i : inputs) { auto id = ++NextReadId_; ReadIds_[r.first].push_back(id); @@ -52,7 +53,12 @@ class TLineageScanner { writer.OnKeyedItem("Name"); writer.OnStringScalar(i.DisplayName); writer.OnKeyedItem("Schema"); - WriteSchema(writer, *r.first->GetTypeAnn()->Cast()->GetItems()[1]->Cast()->GetItemType()->Cast()); + const auto& itemType = *r.first->GetTypeAnn()->Cast()->GetItems()[1]->Cast()->GetItemType()->Cast(); + WriteSchema(writer, itemType, nullptr); + if (formatter.WriteSchemaHeader(writer)) { + WriteSchema(writer, itemType, &formatter); + } + writer.OnEndMap(); } } @@ -63,7 +69,8 @@ class TLineageScanner { for (const auto& w : Writes_) { auto data = w.first->Child(3); TVector outputs; - w.second->GetPlanFormatter().GetOutputs(*w.first, outputs); + auto& formatter = w.second->GetPlanFormatter(); + formatter.GetOutputs(*w.first, outputs); YQL_ENSURE(outputs.size() == 1); auto id = ++NextWriteId_; WriteIds_[w.first] = id; @@ -74,7 +81,12 @@ class TLineageScanner { writer.OnKeyedItem("Name"); writer.OnStringScalar(outputs.front().DisplayName); writer.OnKeyedItem("Schema"); - WriteSchema(writer, *data->GetTypeAnn()->Cast()->GetItemType()->Cast()); + const auto& itemType = *data->GetTypeAnn()->Cast()->GetItemType()->Cast(); + WriteSchema(writer, itemType, nullptr); + if (formatter.WriteSchemaHeader(writer)) { + WriteSchema(writer, itemType, &formatter); + } + writer.OnKeyedItem("Lineage"); auto lineage = CollectLineage(*data); WriteLineage(writer, *lineage); @@ -87,7 +99,7 @@ class TLineageScanner { } private: - void WriteSchema(NYson::TYsonWriter& writer, const TStructExprType& structType) { + void WriteSchema(NYson::TYsonWriter& writer, const TStructExprType& structType, IPlanFormatter* formatter) { writer.OnBeginMap(); for (const auto& i : structType.GetItems()) { if (i->GetName().StartsWith("_yql_sys_")) { @@ -95,7 +107,11 @@ class TLineageScanner { } writer.OnKeyedItem(i->GetName()); - writer.OnStringScalar(FormatType(i->GetItemType())); + if (formatter) { + formatter->WriteTypeDetails(writer, *i->GetItemType()); + } else { + writer.OnStringScalar(FormatType(i->GetItemType())); + } } writer.OnEndMap(); diff --git a/contrib/ydb/library/yql/core/yql_data_provider.h b/contrib/ydb/library/yql/core/yql_data_provider.h index a36255c0ebf..fd2dc28321e 100644 --- a/contrib/ydb/library/yql/core/yql_data_provider.h +++ b/contrib/ydb/library/yql/core/yql_data_provider.h @@ -57,6 +57,9 @@ class IPlanFormatter { virtual void WritePullDetails(const TExprNode& node, NYson::TYsonWriter& writer) = 0; virtual void WritePinDetails(const TExprNode& node, NYson::TYsonWriter& writer) = 0; virtual TString GetOperationDisplayName(const TExprNode& node) = 0; + // returns false if provider schemas aren't supported + virtual bool WriteSchemaHeader(NYson::TYsonWriter& writer) = 0; + virtual void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) = 0; }; class ITrackableNodeProcessor { diff --git a/contrib/ydb/library/yql/dq/integration/yql_dq_integration.h b/contrib/ydb/library/yql/dq/integration/yql_dq_integration.h index 262ebc30d77..2c91c655e91 100644 --- a/contrib/ydb/library/yql/dq/integration/yql_dq_integration.h +++ b/contrib/ydb/library/yql/dq/integration/yql_dq_integration.h @@ -10,10 +10,15 @@ #include #include +#include #include #include +namespace NJson { +class TJsonValue; +} // namespace NJson + namespace NYql { struct TDqSettings; @@ -62,6 +67,11 @@ class IDqIntegration { virtual void Annotate(const TExprNode& node, THashMap& params) = 0; virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap& params, THashMap& secureParams) = 0; virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector& columns, const THashMap& graphParams) = 0; + + // Fill plan operator properties for sources/sinks + // Return true if node was handled + virtual bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap& properties) = 0; + virtual bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap& properties) = 0; }; } // namespace NYql diff --git a/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp b/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp index 24ae595d66a..30741cde640 100644 --- a/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp +++ b/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp @@ -84,4 +84,12 @@ bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode&, TExprCon void TDqIntegrationBase::WriteFullResultTableRef(NYson::TYsonWriter&, const TVector&, const THashMap&) { } +bool TDqIntegrationBase::FillSourcePlanProperties(const NNodes::TExprBase&, TMap&) { + return false; +} + +bool TDqIntegrationBase::FillSinkPlanProperties(const NNodes::TExprBase&, TMap&) { + return false; +} + } // namespace NYql diff --git a/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h b/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h index c2e4823f5fa..4d532816fdd 100644 --- a/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h +++ b/contrib/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h @@ -24,6 +24,8 @@ class TDqIntegrationBase: public IDqIntegration { void Annotate(const TExprNode& node, THashMap& params) override; bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap& params, THashMap& secureParams) override; void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector& columns, const THashMap& graphParams) override; + bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap& properties) override; + bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap& properties) override; protected: bool CanBlockReadTypes(const TStructExprType* node); }; diff --git a/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp index 49a6a4695c4..52693b9e5ea 100644 --- a/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp +++ b/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp @@ -62,6 +62,16 @@ TString TPlanFormatterBase::GetOperationDisplayName(const TExprNode& node) { return TString(node.Content()); } +bool TPlanFormatterBase::WriteSchemaHeader(NYson::TYsonWriter& writer) { + Y_UNUSED(writer); + return false; +} + +void TPlanFormatterBase::WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) { + Y_UNUSED(writer); + Y_UNUSED(type); +} + void TTrackableNodeProcessorBase::GetUsedNodes(const TExprNode& node, TVector& usedNodeIds) { Y_UNUSED(node); usedNodeIds.clear(); diff --git a/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h index bc21ec88b82..10d39e6af29 100644 --- a/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h +++ b/contrib/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h @@ -22,6 +22,8 @@ class TPlanFormatterBase : public IPlanFormatter { void WritePullDetails(const TExprNode& node, NYson::TYsonWriter& writer) override; void WritePinDetails(const TExprNode& node, NYson::TYsonWriter& writer) override; TString GetOperationDisplayName(const TExprNode& node) override; + bool WriteSchemaHeader(NYson::TYsonWriter& writer) override; + void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) override; }; class TTrackableNodeProcessorBase : public ITrackableNodeProcessor { diff --git a/contrib/ydb/library/yql/providers/common/provider/yql_provider.cpp b/contrib/ydb/library/yql/providers/common/provider/yql_provider.cpp index e44e45714e5..e3be8d00971 100644 --- a/contrib/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/contrib/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -227,8 +227,6 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { TMaybeNode columns; TMaybeNode returningList; TMaybeNode primaryKey; - TMaybeNode notNullColumns; - TMaybeNode serialColumns; TMaybeNode partitionBy; TMaybeNode orderBy; TMaybeNode filter; @@ -330,12 +328,6 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { } else if (name == "tableType") { YQL_ENSURE(tuple.Value().Maybe()); tableType = tuple.Value().Cast(); - } else if (name == "notnull") { - YQL_ENSURE(tuple.Value().Maybe()); - notNullColumns = tuple.Value().Cast(); - } else if (name == "serialColumns") { - YQL_ENSURE(tuple.Value().Maybe()); - serialColumns = tuple.Value().Cast(); } else if (name == "pg_delete" || name == "pg_update") { YQL_ENSURE(tuple.Value().Maybe()); pgFilter = tuple.Value().Cast(); @@ -380,8 +372,6 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) { ret.Columns = columns; ret.ReturningList = returningList; ret.PrimaryKey = primaryKey; - ret.NotNullColumns = notNullColumns; - ret.SerialColumns = serialColumns; ret.PartitionBy = partitionBy; ret.OrderBy = orderBy; ret.Filter = filter; diff --git a/contrib/ydb/library/yql/providers/common/provider/yql_provider.h b/contrib/ydb/library/yql/providers/common/provider/yql_provider.h index 0f46aecc9c0..e896918e372 100644 --- a/contrib/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/contrib/ydb/library/yql/providers/common/provider/yql_provider.h @@ -38,8 +38,6 @@ struct TWriteTableSettings { NNodes::TMaybeNode Columns; NNodes::TMaybeNode ReturningList; NNodes::TMaybeNode PrimaryKey; - NNodes::TMaybeNode NotNullColumns; - NNodes::TMaybeNode SerialColumns; NNodes::TMaybeNode PartitionBy; NNodes::TMaybeNode OrderBy; NNodes::TMaybeNode Filter; diff --git a/contrib/ydb/library/yql/providers/dq/actors/compute_actor.cpp b/contrib/ydb/library/yql/providers/dq/actors/compute_actor.cpp index c651a4d780d..481887ab54e 100644 --- a/contrib/ydb/library/yql/providers/dq/actors/compute_actor.cpp +++ b/contrib/ydb/library/yql/providers/dq/actors/compute_actor.cpp @@ -23,7 +23,8 @@ IActor* CreateComputeActor( const TString& computeActorType, const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, ::NMonitoring::TDynamicCounterPtr taskCounters, - NDqProto::EDqStatsMode statsMode) + NDqProto::EDqStatsMode statsMode, + bool enableSpilling) { auto memoryLimits = NDq::TComputeMemoryLimits(); memoryLimits.ChannelBufferSize = 1000000; @@ -40,7 +41,7 @@ IActor* CreateComputeActor( computeRuntimeSettings.ExtraMemoryAllocationPool = 3; computeRuntimeSettings.FailOnUndelivery = false; computeRuntimeSettings.StatsMode = (statsMode != NDqProto::DQ_STATS_MODE_UNSPECIFIED) ? statsMode : NDqProto::DQ_STATS_MODE_FULL; - computeRuntimeSettings.UseSpilling = options.UseSpilling; + computeRuntimeSettings.UseSpilling = enableSpilling; computeRuntimeSettings.AsyncInputPushLimit = 64_MB; // clear fake actorids diff --git a/contrib/ydb/library/yql/providers/dq/actors/compute_actor.h b/contrib/ydb/library/yql/providers/dq/actors/compute_actor.h index 4c184f5f77e..ff9f8bc258e 100644 --- a/contrib/ydb/library/yql/providers/dq/actors/compute_actor.h +++ b/contrib/ydb/library/yql/providers/dq/actors/compute_actor.h @@ -14,6 +14,7 @@ NActors::IActor* CreateComputeActor( const TString& computeActorType, const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, ::NMonitoring::TDynamicCounterPtr taskCounters, - NDqProto::EDqStatsMode statsMode); + NDqProto::EDqStatsMode statsMode, + bool enableSpilling); } // namespace NYql diff --git a/contrib/ydb/library/yql/providers/dq/actors/executer_actor.cpp b/contrib/ydb/library/yql/providers/dq/actors/executer_actor.cpp index cc6917cc1bf..128fc3d00ab 100644 --- a/contrib/ydb/library/yql/providers/dq/actors/executer_actor.cpp +++ b/contrib/ydb/library/yql/providers/dq/actors/executer_actor.cpp @@ -188,6 +188,7 @@ class TDqExecuter: public TRichActor, NYql::TCounters { const TString computeActorType = Settings->ComputeActorType.Get().GetOrElse("sync"); + bool enableSpilling = Settings->SpillingEngine.Get().GetOrElse(TDqSettings::TDefault::SpillingEngine) != TDqSettings::ESpillingEngine::Disable; auto resourceAllocator = RegisterChild(CreateResourceAllocator( GwmActorId, SelfId(), ControlId, workerCount, @@ -201,6 +202,7 @@ class TDqExecuter: public TRichActor, NYql::TCounters { allocateRequest->Record.SetCreateComputeActor(enableComputeActor); allocateRequest->Record.SetComputeActorType(computeActorType); allocateRequest->Record.SetStatsMode(StatsMode); + allocateRequest->Record.SetEnableSpilling(enableSpilling); if (enableComputeActor) { ActorIdToProto(ControlId, allocateRequest->Record.MutableResultActorId()); } diff --git a/contrib/ydb/library/yql/providers/dq/api/protos/dqs.proto b/contrib/ydb/library/yql/providers/dq/api/protos/dqs.proto index 1c92d6f9637..a069b6c5e3e 100644 --- a/contrib/ydb/library/yql/providers/dq/api/protos/dqs.proto +++ b/contrib/ydb/library/yql/providers/dq/api/protos/dqs.proto @@ -39,6 +39,7 @@ message TAllocateWorkersRequest { uint64 FreeWorkerAfterMs = 14; NYql.NDqProto.EDqStatsMode StatsMode = 16; + bool EnableSpilling = 17; // false } message TWorkerGroup { diff --git a/contrib/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp b/contrib/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp index 1a59a485e73..80061126dc8 100644 --- a/contrib/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp +++ b/contrib/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp @@ -24,7 +24,7 @@ using namespace NKikimr::NMiniKQL; namespace { -const THashSet VALID_SOURCES = {DqProviderName, ConfigProviderName, YtProviderName, ClickHouseProviderName, YdbProviderName, S3ProviderName}; +const THashSet VALID_SOURCES = {DqProviderName, ConfigProviderName, YtProviderName, ClickHouseProviderName, YdbProviderName, S3ProviderName, PgProviderName}; const THashSet VALID_SINKS = {ResultProviderName, YtProviderName, S3ProviderName}; } diff --git a/contrib/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp b/contrib/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp index 6f45ec18a0b..1ea923ef917 100644 --- a/contrib/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp +++ b/contrib/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp @@ -219,6 +219,7 @@ class TLocalWorkerManager: public TWorkerManagerCommon { } bool createComputeActor = ev->Get()->Record.GetCreateComputeActor(); TString computeActorType = ev->Get()->Record.GetComputeActorType(); + bool enableSpilling = Options.UseSpilling && ev->Get()->Record.GetEnableSpilling(); if (createComputeActor && !Options.CanUseComputeActor) { Send(ev->Sender, MakeHolder("Compute Actor Disabled", NYql::NDqProto::StatusIds::BAD_REQUEST), 0, ev->Cookie); @@ -296,14 +297,15 @@ class TLocalWorkerManager: public TWorkerManagerCommon { computeActorType, Options.TaskRunnerActorFactory, taskCounters, - ev->Get()->Record.GetStatsMode())); + ev->Get()->Record.GetStatsMode(), + enableSpilling)); } else { actor.Reset(CreateWorkerActor( Options.RuntimeData, traceId, Options.TaskRunnerActorFactory, Options.AsyncIoFactory, - Options.UseSpilling)); + enableSpilling)); } allocationInfo.WorkerActors.emplace_back(RegisterChild( actor.Release(), createComputeActor ? NYql::NDq::TEvDq::TEvAbortExecution::Unavailable("Aborted by LWM").Release() : nullptr diff --git a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.darwin-x86_64.txt b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.darwin-x86_64.txt index 655c5b80410..a48e38bad50 100644 --- a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.darwin-x86_64.txt +++ b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.darwin-x86_64.txt @@ -39,6 +39,7 @@ target_link_libraries(providers-generic-provider PUBLIC providers-generic-expr_nodes providers-generic-proto generic-connector-libcpp + yql-utils-plan ) target_sources(providers-generic-provider PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp diff --git a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-aarch64.txt b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-aarch64.txt index ba3b3f735c5..34383894e4f 100644 --- a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-aarch64.txt +++ b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-aarch64.txt @@ -40,6 +40,7 @@ target_link_libraries(providers-generic-provider PUBLIC providers-generic-expr_nodes providers-generic-proto generic-connector-libcpp + yql-utils-plan ) target_sources(providers-generic-provider PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp diff --git a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-x86_64.txt b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-x86_64.txt index ba3b3f735c5..34383894e4f 100644 --- a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-x86_64.txt +++ b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.linux-x86_64.txt @@ -40,6 +40,7 @@ target_link_libraries(providers-generic-provider PUBLIC providers-generic-expr_nodes providers-generic-proto generic-connector-libcpp + yql-utils-plan ) target_sources(providers-generic-provider PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp diff --git a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.windows-x86_64.txt b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.windows-x86_64.txt index 655c5b80410..a48e38bad50 100644 --- a/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.windows-x86_64.txt +++ b/contrib/ydb/library/yql/providers/generic/provider/CMakeLists.windows-x86_64.txt @@ -39,6 +39,7 @@ target_link_libraries(providers-generic-provider PUBLIC providers-generic-expr_nodes providers-generic-proto generic-connector-libcpp + yql-utils-plan ) target_sources(providers-generic-provider PRIVATE ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp diff --git a/contrib/ydb/library/yql/providers/generic/provider/ya.make b/contrib/ydb/library/yql/providers/generic/provider/ya.make index 15a444565b6..efc206622da 100644 --- a/contrib/ydb/library/yql/providers/generic/provider/ya.make +++ b/contrib/ydb/library/yql/providers/generic/provider/ya.make @@ -51,6 +51,7 @@ PEERDIR( contrib/ydb/library/yql/providers/generic/expr_nodes contrib/ydb/library/yql/providers/generic/proto contrib/ydb/library/yql/providers/generic/connector/libcpp + contrib/ydb/library/yql/utils/plan ) END() diff --git a/contrib/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp b/contrib/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp index 3a1d7fd7a49..667665e0d0a 100644 --- a/contrib/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp +++ b/contrib/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace NYql { @@ -94,8 +95,8 @@ namespace NYql { void FillSourceSettings(const TExprNode& node, ::google::protobuf::Any& protoSettings, TString& sourceType) override { const TDqSource source(&node); - if (const auto maySettings = source.Settings().Maybe()) { - const auto settings = maySettings.Cast(); + if (const auto maybeSettings = source.Settings().Maybe()) { + const auto settings = maybeSettings.Cast(); const auto& clusterName = source.DataSource().Cast().Cluster().StringValue(); const auto& table = settings.Table().StringValue(); const auto& token = settings.Token().Name().StringValue(); @@ -168,6 +169,67 @@ namespace NYql { } } + bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap& properties) override { + if (!node.Maybe()) { + return false; + } + + auto source = node.Cast(); + if (!source.Settings().Maybe()) { + return false; + } + + const TGenSourceSettings settings = source.Settings().Cast(); + const TString& clusterName = source.DataSource().Cast().Cluster().StringValue(); + const TString& table = settings.Table().StringValue(); + properties["Table"] = table; + auto [tableMeta, issue] = State_->GetTable(clusterName, table); + if (!issue) { + const NConnector::NApi::TDataSourceInstance& dataSourceInstance = tableMeta.value()->DataSourceInstance; + switch (dataSourceInstance.kind()) { + case NConnector::NApi::CLICKHOUSE: + properties["SourceType"] = "ClickHouse"; + break; + case NConnector::NApi::POSTGRESQL: + properties["SourceType"] = "PostgreSql"; + break; + case NConnector::NApi::DATA_SOURCE_KIND_UNSPECIFIED: + break; + default: + properties["SourceType"] = NConnector::NApi::EDataSourceKind_Name(dataSourceInstance.kind()); + break; + } + + if (const TString& database = dataSourceInstance.database()) { + properties["Database"] = database; + } + + switch (dataSourceInstance.protocol()) { + case NConnector::NApi::NATIVE: + properties["Protocol"] = "Native"; + break; + case NConnector::NApi::HTTP: + properties["Protocol"] = "Http"; + break; + case NConnector::NApi::PROTOCOL_UNSPECIFIED: + break; + default: + properties["Protocol"] = NConnector::NApi::EProtocol_Name(dataSourceInstance.protocol()); + break; + } + } + if (settings.Columns().Size()) { + auto& columns = properties["ReadColumns"]; + for (const TCoAtom col : settings.Columns()) { + columns.AppendValue(col.StringValue()); + } + } + if (auto predicate = settings.FilterPredicate(); !IsEmptyFilterPredicate(predicate)) { + properties["Filter"] = NPlanUtils::PrettyExprStr(predicate); + } + return true; + } + void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override { RegisterDqGenericMkqlCompilers(compiler, State_); } diff --git a/contrib/ydb/library/yql/providers/pg/provider/ya.make b/contrib/ydb/library/yql/providers/pg/provider/ya.make index 2d557b325ed..f7288fdd5d8 100644 --- a/contrib/ydb/library/yql/providers/pg/provider/ya.make +++ b/contrib/ydb/library/yql/providers/pg/provider/ya.make @@ -5,6 +5,7 @@ SRCS( yql_pg_datasink_execution.cpp yql_pg_datasink_type_ann.cpp yql_pg_datasource.cpp + yql_pg_dq_integration.cpp yql_pg_datasource_type_ann.cpp yql_pg_provider.cpp yql_pg_provider.h @@ -16,6 +17,8 @@ YQL_LAST_ABI_VERSION() PEERDIR( contrib/ydb/library/yql/core contrib/ydb/library/yql/core/type_ann + contrib/ydb/library/yql/dq/integration + contrib/ydb/library/yql/providers/common/dq contrib/ydb/library/yql/providers/common/provider contrib/ydb/library/yql/providers/common/transform contrib/ydb/library/yql/providers/pg/expr_nodes diff --git a/contrib/ydb/library/yql/providers/pg/provider/yql_pg_datasource.cpp b/contrib/ydb/library/yql/providers/pg/provider/yql_pg_datasource.cpp index 123906364db..8e63dcccaf4 100644 --- a/contrib/ydb/library/yql/providers/pg/provider/yql_pg_datasource.cpp +++ b/contrib/ydb/library/yql/providers/pg/provider/yql_pg_datasource.cpp @@ -17,6 +17,7 @@ class TPgDataSourceImpl : public TDataProviderBase { TPgDataSourceImpl(TPgState::TPtr state) : State_(state) , TypeAnnotationTransformer_(CreatePgDataSourceTypeAnnotationTransformer(state)) + , DqIntegration_(CreatePgDqIntegration(State_)) {} TStringBuf GetName() const override { @@ -110,9 +111,14 @@ class TPgDataSourceImpl : public TDataProviderBase { return false; } + IDqIntegration* GetDqIntegration() override { + return DqIntegration_.Get(); + } + private: TPgState::TPtr State_; const THolder TypeAnnotationTransformer_; + const THolder DqIntegration_; }; TIntrusivePtr CreatePgDataSource(TPgState::TPtr state) { diff --git a/contrib/ydb/library/yql/providers/pg/provider/yql_pg_dq_integration.cpp b/contrib/ydb/library/yql/providers/pg/provider/yql_pg_dq_integration.cpp new file mode 100644 index 00000000000..e517e58cec5 --- /dev/null +++ b/contrib/ydb/library/yql/providers/pg/provider/yql_pg_dq_integration.cpp @@ -0,0 +1,44 @@ +#include "yql_pg_provider_impl.h" +#include +#include + +namespace NYql { + +using namespace NNodes; + +namespace { + +class TPgDqIntegration: public TDqIntegrationBase { +public: + TPgDqIntegration(TPgState::TPtr state) + : State_(state) + {} + + bool CanRead(const TExprNode& read, TExprContext&, bool) override { + return TPgReadTable::Match(&read); + } + + TMaybe EstimateReadSize(ui64 /*dataSizePerJob*/, ui32 /*maxTasksPerStage*/, const TVector& read, TExprContext&) override { + if (AllOf(read, [](const auto val) { return TPgReadTable::Match(val); })) { + return 0ul; + } + return Nothing(); + } + + ui64 Partition(const TDqSettings&, size_t, const TExprNode&, TVector& partitions, TString*, TExprContext&, bool) override { + partitions.clear(); + partitions.emplace_back(); + return 0ULL; + } + +private: + const TPgState::TPtr State_; +}; + +} + +THolder CreatePgDqIntegration(TPgState::TPtr state) { + return MakeHolder(state); +} + +} diff --git a/contrib/ydb/library/yql/providers/pg/provider/yql_pg_provider_impl.h b/contrib/ydb/library/yql/providers/pg/provider/yql_pg_provider_impl.h index b48f34dafc2..8fab652f1c9 100644 --- a/contrib/ydb/library/yql/providers/pg/provider/yql_pg_provider_impl.h +++ b/contrib/ydb/library/yql/providers/pg/provider/yql_pg_provider_impl.h @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -17,5 +18,6 @@ THolder CreatePgDataSinkTypeAnnotationTransformer(TPgSt THolder CreatePgDataSinkExecTransformer(TPgState::TPtr state); +THolder CreatePgDqIntegration(TPgState::TPtr state); } diff --git a/contrib/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/contrib/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp index 8ec6f3b485b..541d8bf3154 100644 --- a/contrib/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp +++ b/contrib/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp @@ -1863,7 +1863,8 @@ class TS3ReadCoroImpl : public TActorCoroImpl { readers[readyReaderIndex].reset(); } if (isCancelled) { - LOG_CORO_D("RunCoroBlockArrowParserOverHttp - STOPPED ON SATURATION"); + LOG_CORO_D("RunCoroBlockArrowParserOverHttp - STOPPED ON SATURATION, downloaded " << + QueueBufferCounter->DownloadedBytes << " bytes"); break; } } diff --git a/contrib/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json b/contrib/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json index ea5b33c99dc..90df231ec91 100644 --- a/contrib/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json +++ b/contrib/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json @@ -94,8 +94,7 @@ "Children": [ {"Index": 0, "Name": "Paths", "Type": "TS3Paths"}, {"Index": 1, "Name": "Format", "Type": "TCoAtom"}, - {"Index": 2, "Name": "RowsLimitHint", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Settings", "Type": "TExprBase", "Optional": true} + {"Index": 2, "Name": "Settings", "Type": "TExprBase", "Optional": true} ] }, { diff --git a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp index c9faffe43ed..c9e8d50f80a 100644 --- a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp +++ b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp @@ -311,6 +311,10 @@ class TS3DataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { return TStatus::Error; } + if (!EnsureAtom(*input->Child(TS3SourceSettings::idx_RowsLimitHint), ctx)) { + return TStatus::Error; + } + const TTypeAnnotationNode* itemType = ctx.MakeType(EDataSlot::String); if (extraColumnsType->GetSize()) { itemType = ctx.MakeType( @@ -510,7 +514,7 @@ class TS3DataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { } TStatus HandleObject(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureMinMaxArgsCount(*input, 2U, 4U, ctx)) { + if (!EnsureMinMaxArgsCount(*input, 2U, 3U, ctx)) { return TStatus::Error; } @@ -524,10 +528,6 @@ class TS3DataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { return TStatus::Error; } - if (!EnsureAtom(*input->Child(TS3SourceSettings::idx_RowsLimitHint), ctx)) { - return TStatus::Error; - } - if (input->ChildrenSize() > TS3Object::idx_Settings) { bool haveProjection = false; bool havePartitionedBy = false; diff --git a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp index 9aa62824712..33014877ef9 100644 --- a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp +++ b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp @@ -14,6 +14,8 @@ #include #include +#include + namespace NYql { using namespace NNodes; @@ -228,7 +230,7 @@ class TS3DqIntegration: public TDqIntegrationBase { .Token() .Name().Build(token) .Build() - .RowsLimitHint(s3ReadObject.Object().RowsLimitHint()) + .RowsLimitHint(ctx.NewAtom(read->Pos(), "")) .Format(s3ReadObject.Object().Format()) .RowType(ExpandType(s3ReadObject.Pos(), *rowType, ctx)) .Settings(s3ReadObject.Object().Settings()) @@ -271,7 +273,7 @@ class TS3DqIntegration: public TDqIntegrationBase { .Token() .Name().Build(token) .Build() - .RowsLimitHint(s3ReadObject.Object().RowsLimitHint()) + .RowsLimitHint(ctx.NewAtom(read->Pos(), "")) .SizeLimit( sizeLimitIndex != -1 ? readSettings->Child(sizeLimitIndex)->TailPtr() : emptyNode) @@ -412,6 +414,63 @@ class TS3DqIntegration: public TDqIntegrationBase { } } + bool FillSourcePlanProperties(const NNodes::TExprBase& node, TMap& properties) override { + if (!node.Maybe()) { + return false; + } + + auto source = node.Cast(); + if (auto maybeSettings = source.Settings().Maybe()) { + const TS3SourceSettings settings = maybeSettings.Cast(); + properties["Name"] = "Raw read from external data source"; + properties["Format"] = "raw"; + if (TString limit = settings.RowsLimitHint().StringValue()) { + properties["RowsLimitHint"] = limit; + } + return true; + } else if (auto maybeSettings = source.Settings().Maybe()) { + const TS3ParseSettings settings = maybeSettings.Cast(); + properties["Name"] = "Parse from external data source"; + properties["Format"] = settings.Format().StringValue(); + if (TString limit = settings.RowsLimitHint().StringValue()) { + properties["RowsLimitHint"] = limit; + } + + const TStructExprType* fullRowType = settings.RowType().Ref().GetTypeAnn()->Cast()->GetType()->Cast(); + auto rowTypeItems = fullRowType->GetItems(); + auto& columns = properties["ReadColumns"]; + for (auto& item : rowTypeItems) { + columns.AppendValue(item->GetName()); + } + return true; + } + return false; + } + + bool FillSinkPlanProperties(const NNodes::TExprBase& node, TMap& properties) override { + if (!node.Maybe()) { + return false; + } + + auto sink = node.Cast(); + if (auto maybeS3SinkSettings = sink.Settings().Maybe()) { + auto s3SinkSettings = maybeS3SinkSettings.Cast(); + properties["Extension"] = s3SinkSettings.Extension().StringValue(); + if (auto settingsList = s3SinkSettings.Settings().Maybe()) { + for (const TExprNode::TPtr& s : s3SinkSettings.Settings().Raw()->Children()) { + if (s->ChildrenSize() >= 2 && s->Child(0)->Content() == "compression"sv) { + auto val = s->Child(1)->Content(); + if (val) { + properties["Compression"] = TString(val); + } + } + } + } + return true; + } + return false; + } + void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override { RegisterDqS3MkqlCompilers(compiler, State_); } diff --git a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp index 7bfc226d188..247d648f958 100644 --- a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp +++ b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp @@ -496,7 +496,6 @@ class TS3IODiscoveryTransformer : public TGraphTransformerBase { s3Object = Build(ctx, object.Pos()) .Paths(ctx.NewList(object.Pos(), std::move(pathNodes))) .Format(std::move(format)) - .RowsLimitHint(ctx.NewAtom(object.Pos(), "")) .Settings(ctx.NewList(object.Pos(), std::move(settings))) .Done().Ptr(); diff --git a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_logical_opt.cpp b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_logical_opt.cpp index 62eb0496bee..bdb3f6c14ac 100644 --- a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_logical_opt.cpp +++ b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_logical_opt.cpp @@ -171,6 +171,7 @@ class TS3LogicalOptProposalTransformer : public TOptimizeTransformerBase { AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembersOverDqSource)); AddHandler(0, &TDqSourceWrap::Match, HNDL(MergeS3Paths)); AddHandler(0, &TDqSourceWrap::Match, HNDL(CleanupExtraColumns)); + AddHandler(0, &TCoTake::Match, HNDL(PushDownLimit)); #undef HNDL } @@ -661,6 +662,56 @@ class TS3LogicalOptProposalTransformer : public TOptimizeTransformerBase { } + template + TMaybeNode ConstructNodeForPushDownLimit(TCoTake take, TDqSourceWrap source, TSettings settings, TCoUint64 count, TExprContext& ctx) const { + return Build(ctx, take.Pos()) + .InitFrom(take) + .Input() + .InitFrom(source) + .Input() + .InitFrom(settings) + .RowsLimitHint(count.Literal()) + .Build() + .Build() + .Done(); + } + + TMaybeNode PushDownLimit(TExprBase node, TExprContext& ctx) const { + auto take = node.Cast(); + + auto maybeSource = take.Input().Maybe(); + auto maybeSettings = maybeSource.Input().Maybe(); + if (!maybeSettings) { + return take; + } + YQL_CLOG(TRACE, ProviderS3) << "Trying to push down limit for S3"; + + auto maybeCount = take.Count().Maybe(); + if (!maybeCount) { + return take; + } + auto count = maybeCount.Cast(); + auto countNum = FromString(count.Literal().Ref().Content()); + YQL_ENSURE(countNum > 0, "Got invalid limit " << countNum << " to push down"); + + auto settings = maybeSettings.Cast(); + if (auto rowsLimitHintStr = settings.RowsLimitHint().Ref().Content(); !rowsLimitHintStr.empty()) { + // LimitHint is already pushed down + auto rowsLimitHint = FromString(rowsLimitHintStr); + if (countNum >= rowsLimitHint) { + // Already propagated + return node; + } + } + + if (auto sourceSettings = maybeSettings.Maybe()) { + return ConstructNodeForPushDownLimit(take, maybeSource.Cast(), sourceSettings.Cast(), count, ctx); + } + auto parseSettings = maybeSettings.Maybe(); + YQL_ENSURE(parseSettings, "unsupported type derived from TS3SourceSettingsBase"); + return ConstructNodeForPushDownLimit(take, maybeSource.Cast(), parseSettings.Cast(), count, ctx); + } + private: const TS3State::TPtr State_; }; diff --git a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_phy_opt.cpp b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_phy_opt.cpp index bf416649203..00fc74d8d8d 100644 --- a/contrib/ydb/library/yql/providers/s3/provider/yql_s3_phy_opt.cpp +++ b/contrib/ydb/library/yql/providers/s3/provider/yql_s3_phy_opt.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -97,7 +98,6 @@ class TS3PhysicalOptProposalTransformer : public TOptimizeTransformerBase { AddHandler(0, &TS3WriteObject::Match, HNDL(S3WriteObject)); } AddHandler(0, &TS3Insert::Match, HNDL(S3Insert)); - AddHandler(0, &TCoTake::Match, HNDL(PushDownLimit)); #undef HNDL } @@ -112,49 +112,6 @@ class TS3PhysicalOptProposalTransformer : public TOptimizeTransformerBase { return TExprBase(maybeRead.Cast().World().Ptr()); } - TMaybeNode PushDownLimit(TExprBase node, TExprContext& ctx) const { - auto take = node.Cast(); - - auto maybeReadObject = take - .Input().Maybe() - .Input().Maybe(); - if (!maybeReadObject) { - return node; - } - - auto maybeCount = take.Count().Maybe(); - if (!maybeCount) { - return node; - } - auto count = maybeCount.Cast(); - auto countNum = FromString(count.Literal().Ref().Content()); - YQL_ENSURE(countNum > 0, "Got invalid limit " << countNum << " to push down"); - - auto object = maybeReadObject.Cast().Object(); - if (auto rowsLimitHintStr = object.RowsLimitHint().Ref().Content(); !rowsLimitHintStr.empty()) { - // LimitHint is already pushed down - auto rowsLimitHint = FromString(rowsLimitHintStr); - if (countNum >= rowsLimitHint) { - // Already propagated - return node; - } - } - - return Build(ctx, take.Pos()) - .InitFrom(take) - .Input() - .Input() - .InitFrom(maybeReadObject.Cast()) - .Object() - .InitFrom(object) - .RowsLimitHint(count.Literal()) - .Build() - .Build() - .Build() - .Count(count.Ptr()) - .Done(); - } - TMaybe BuildSinkStage(TPositionHandle writePos, TS3DataSink dataSink, TS3Target target, TExprBase input, TExprContext& ctx, const TGetParents& getParents) const { const auto& cluster = dataSink.Cluster().StringValue(); const auto token = "cluster:default_" + cluster; diff --git a/contrib/ydb/library/yql/providers/yt/lib/schema/schema.cpp b/contrib/ydb/library/yql/providers/yt/lib/schema/schema.cpp index 1779103c66a..9d61057ea03 100644 --- a/contrib/ydb/library/yql/providers/yt/lib/schema/schema.cpp +++ b/contrib/ydb/library/yql/providers/yt/lib/schema/schema.cpp @@ -2,11 +2,13 @@ #include #include +#include #include #include #include +#include #include #include @@ -1017,4 +1019,11 @@ NYT::TSortColumns ToYTSortColumns(const TVector>& sortC return res; } +TString GetTypeV3String(const TTypeAnnotationNode& type, ui64 nativeTypeCompatibility) { + NYT::TNode typeNode; + NYT::TNodeBuilder nodeBuilder(&typeNode); + NCommon::WriteTypeToYson(nodeBuilder, &type); + return NYT::NodeToCanonicalYsonString(RowSpecYqlTypeToYtNativeType(typeNode, nativeTypeCompatibility)); +} + } // NYql diff --git a/contrib/ydb/library/yql/providers/yt/lib/schema/schema.h b/contrib/ydb/library/yql/providers/yt/lib/schema/schema.h index 694533ea697..0222636ec73 100644 --- a/contrib/ydb/library/yql/providers/yt/lib/schema/schema.h +++ b/contrib/ydb/library/yql/providers/yt/lib/schema/schema.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -12,6 +14,8 @@ namespace NYql { +class TTypeAnnotationNode; + struct TYTSortInfo { TVector> Keys; bool Unique = false; @@ -26,5 +30,6 @@ bool ValidateTableSchema(const TString& tableName, const NYT::TNode& attributes, void MergeInferredSchemeWithSort(NYT::TNode& schema, TYTSortInfo& sortInfo); NYT::TTableSchema RowSpecToYTSchema(const NYT::TNode& rowSpec, ui64 nativeTypeCompatibility); NYT::TSortColumns ToYTSortColumns(const TVector>& sortColumns); +TString GetTypeV3String(const TTypeAnnotationNode& type, ui64 nativeTypeCompatibility = NTCF_ALL); } // NYql diff --git a/contrib/ydb/library/yql/providers/yt/lib/schema/ya.make b/contrib/ydb/library/yql/providers/yt/lib/schema/ya.make index 1e58a3f8f5a..b1ca4fdc821 100644 --- a/contrib/ydb/library/yql/providers/yt/lib/schema/ya.make +++ b/contrib/ydb/library/yql/providers/yt/lib/schema/ya.make @@ -10,6 +10,7 @@ PEERDIR( contrib/ydb/library/yql/utils contrib/ydb/library/yql/utils/log contrib/ydb/library/yql/providers/common/codec + contrib/ydb/library/yql/providers/common/schema/expr contrib/ydb/library/yql/providers/yt/common ) diff --git a/contrib/ydb/library/yql/providers/yt/provider/ya.make b/contrib/ydb/library/yql/providers/yt/provider/ya.make index 680e35a2241..d7e4c491077 100644 --- a/contrib/ydb/library/yql/providers/yt/provider/ya.make +++ b/contrib/ydb/library/yql/providers/yt/provider/ya.make @@ -88,6 +88,7 @@ PEERDIR( contrib/ydb/library/yql/providers/yt/lib/mkql_helpers contrib/ydb/library/yql/providers/yt/lib/res_pull contrib/ydb/library/yql/providers/yt/lib/row_spec + contrib/ydb/library/yql/providers/yt/lib/schema contrib/ydb/library/yql/providers/yt/lib/skiff contrib/ydb/library/yql/providers/yt/lib/yson_helpers contrib/ydb/library/yql/providers/yt/opt diff --git a/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp b/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp index a9db606d8b2..561d2d39aba 100644 --- a/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp +++ b/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasink.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -499,6 +500,15 @@ class TYtDataSink : public TDataProviderBase { return TString{node.Content()}; } + bool WriteSchemaHeader(NYson::TYsonWriter& writer) override { + writer.OnKeyedItem("YtSchema"); + return true; + } + + void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) override { + writer.OnStringScalar(GetTypeV3String(type)); + } + ITrackableNodeProcessor& GetTrackableNodeProcessor() override { return *TrackableNodeProcessor_; } diff --git a/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp b/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp index 13e58409755..9b66493a506 100644 --- a/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp +++ b/contrib/ydb/library/yql/providers/yt/provider/yql_yt_datasource.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -433,6 +434,15 @@ class TYtDataSource : public TDataProviderBase { } } + bool WriteSchemaHeader(NYson::TYsonWriter& writer) override { + writer.OnKeyedItem("YtSchema"); + return true; + } + + void WriteTypeDetails(NYson::TYsonWriter& writer, const TTypeAnnotationNode& type) override { + writer.OnStringScalar(GetTypeV3String(type)); + } + ITrackableNodeProcessor& GetTrackableNodeProcessor() override { return *TrackableNodeProcessor_; } diff --git a/contrib/ydb/library/yql/sql/pg/pg_sql.cpp b/contrib/ydb/library/yql/sql/pg/pg_sql.cpp index 6161a1ccee6..246b5b86752 100644 --- a/contrib/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/contrib/ydb/library/yql/sql/pg/pg_sql.cpp @@ -1462,16 +1462,21 @@ class TConverter : public IPGParseEvents { #pragma region CreateTable private: + + struct TColumnInfo { + TString Name; + TString Type; + bool Serial = false; + bool NotNull = false; + TAstNode* Default = nullptr; + }; + struct TCreateTableCtx { - std::vector Columns; - std::unordered_set ColumnsSet; + std::unordered_map ColumnsSet; + std::vector ColumnOrder; std::vector PrimaryKey; - std::vector NotNullColumns; std::vector> UniqConstr; - std::unordered_set NotNullColSet; bool isTemporary; - std::vector SerialColumns; - std::unordered_map Defaults; bool ifNotExists; }; @@ -1519,11 +1524,12 @@ class TConverter : public IPGParseEvents { auto node = ListNodeNth(pk->keys, i); auto nodeName = StrVal(node); - if (!ctx.ColumnsSet.contains(nodeName)) { + auto it = ctx.ColumnsSet.find(nodeName); + if (it == ctx.ColumnsSet.end()) { AddError("PK column does not belong to table"); return false; } - AddNonNullColumn(ctx, nodeName); + it->second.NotNull = true; ctx.PrimaryKey.push_back(QA(StrVal(node))); } @@ -1557,14 +1563,6 @@ class TConverter : public IPGParseEvents { return true; } - bool AddNonNullColumn(TCreateTableCtx& ctx, const char* colName) { - auto [it, inserted] = ctx.NotNullColSet.insert(colName); - if (inserted) - ctx.NotNullColumns.push_back(QA(colName)); - - return inserted; - } - const TString& FindColumnTypeAlias(const TString& colType, bool& isTypeSerial) { const static std::unordered_map aliasMap { {"smallserial", "int2"}, @@ -1584,6 +1582,8 @@ class TConverter : public IPGParseEvents { } bool AddColumn(TCreateTableCtx& ctx, const ColumnDef* node) { + TColumnInfo cinfo{.Name = node->colname}; + if (node->constraints) { for (ui32 i = 0; i < ListLength(node->constraints); ++i) { auto constraintNode = @@ -1591,7 +1591,7 @@ class TConverter : public IPGParseEvents { switch (constraintNode->contype) { case CONSTR_NOTNULL: - AddNonNullColumn(ctx, node->colname); + cinfo.NotNull = true; break; case CONSTR_PRIMARY: { @@ -1599,7 +1599,7 @@ class TConverter : public IPGParseEvents { AddError("Only a single PK is allowed per table"); return false; } - AddNonNullColumn(ctx, node->colname); + cinfo.NotNull = true; ctx.PrimaryKey.push_back(QA(node->colname)); } break; @@ -1611,11 +1611,10 @@ class TConverter : public IPGParseEvents { TExprSettings settings; settings.AllowColumns = false; settings.Scope = "DEFAULT"; - auto expr = ParseExpr(constraintNode->raw_expr, settings); - if (!expr) { + cinfo.Default = ParseExpr(constraintNode->raw_expr, settings); + if (!cinfo.Default) { return false; } - ctx.Defaults[node->colname] = expr; } break; default: @@ -1624,26 +1623,19 @@ class TConverter : public IPGParseEvents { } } } - auto [it, inserted] = ctx.ColumnsSet.insert(node->colname); - if (!inserted) { - AddError("duplicated column names found"); - return false; - } // for now we pass just the last part of the type name auto colTypeVal = StrVal( ListNodeNth(node->typeName->names, ListLength(node->typeName->names) - 1)); - bool isTypeSerial = false; - const auto colType = FindColumnTypeAlias(colTypeVal, isTypeSerial); - if (isTypeSerial) { - ctx.SerialColumns.push_back(QA(node->colname)); + cinfo.Type = FindColumnTypeAlias(colTypeVal, cinfo.Serial); + auto [it, inserted] = ctx.ColumnsSet.emplace(node->colname, cinfo); + if (!inserted) { + AddError("duplicated column names found"); + return false; } - ctx.Columns.push_back( - QL(QA(node->colname), L(A("PgType"), QA(colType))) - ); - + ctx.ColumnOrder.push_back(node->colname); return true; } @@ -1675,21 +1667,43 @@ class TConverter : public IPGParseEvents { return true; } + TAstNode* BuildColumnsOptions(TCreateTableCtx& ctx) { + std::vector columns; + + for(const auto& name: ctx.ColumnOrder) { + auto it = ctx.ColumnsSet.find(name); + Y_ENSURE(it != ctx.ColumnsSet.end()); + + const auto& cinfo = it->second; + + std::vector constraints; + if (cinfo.Serial) { + constraints.push_back(QL(QA("serial"))); + } + + if (cinfo.NotNull) { + constraints.push_back(QL(QA("not_null"))); + } + + if (cinfo.Default) { + constraints.push_back(QL(QA("default"), cinfo.Default)); + } + + columns.push_back(QL(QA(cinfo.Name), L(A("PgType"), QA(cinfo.Type)), QL(QA("columnConstraints"), QVL(constraints.data(), constraints.size())))); + } + + return QVL(columns.data(), columns.size()); + } + TAstNode* BuildCreateTableOptions(TCreateTableCtx& ctx) { std::vector options; TString mode = (ctx.ifNotExists) ? "create_if_not_exists" : "create"; options.push_back(QL(QA("mode"), QA(mode))); - options.push_back(QL(QA("columns"), QVL(ctx.Columns.data(), ctx.Columns.size()))); + options.push_back(QL(QA("columns"), BuildColumnsOptions(ctx))); if (!ctx.PrimaryKey.empty()) { options.push_back(QL(QA("primarykey"), QVL(ctx.PrimaryKey.data(), ctx.PrimaryKey.size()))); } - if (!ctx.NotNullColumns.empty()) { - options.push_back(QL(QA("notnull"), QVL(ctx.NotNullColumns.data(), ctx.NotNullColumns.size()))); - } - if (!ctx.SerialColumns.empty()) { - options.push_back(QL(QA("serialColumns"), QVL(ctx.SerialColumns.data(), ctx.SerialColumns.size()))); - } for (auto& uniq : ctx.UniqConstr) { auto columns = QVL(uniq.data(), uniq.size()); options.push_back(QL(QA("index"), QL( @@ -1698,9 +1712,6 @@ class TConverter : public IPGParseEvents { QL(QA("dataColumns"), QL()), QL(QA("indexColumns"), columns)))); } - for (auto& def : ctx.Defaults) { - options.push_back(QL(QA("default"), QA(def.first), def.second)); - } if (ctx.isTemporary) { options.push_back(QL(QA("temporary"))); } diff --git a/contrib/ydb/library/yql/sql/pg/pg_sql_ut.cpp b/contrib/ydb/library/yql/sql/pg/pg_sql_ut.cpp index fe86027f858..fc80695c2fc 100644 --- a/contrib/ydb/library/yql/sql/pg/pg_sql_ut.cpp +++ b/contrib/ydb/library/yql/sql/pg/pg_sql_ut.cpp @@ -94,7 +94,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'text))))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '())) '('b (PgType 'text) '('columnConstraints '()))))))) (let world (CommitAll! world)) (return world) ) @@ -110,7 +110,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'text)))) '('notnull '('a))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('not_null)))) '('b (PgType 'text) '('columnConstraints '()))))))) (let world (CommitAll! world)) (return world) ) @@ -126,7 +126,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'text)))) '('primarykey '('a)) '('notnull '('a))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('not_null)))) '('b (PgType 'text) '('columnConstraints '())))) '('primarykey '('a))))) (let world (CommitAll! world)) (return world) ) @@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'int4)))) '('primarykey '('a)) '('notnull '('a)) '('default 'b (PgConst '0 (PgType 'int4)))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('not_null)))) '('b (PgType 'int4) '('columnConstraints '('('default (PgConst '0 (PgType 'int4)))))))) '('primarykey '('a))))) (let world (CommitAll! world)) (return world) ) @@ -158,7 +158,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'text)))) '('primarykey '('a)) '('notnull '('a))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('not_null)))) '('b (PgType 'text) '('columnConstraints '())))) '('primarykey '('a))))) (let world (CommitAll! world)) (return world) ) @@ -174,7 +174,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'text)))) '('primarykey '('a)) '('notnull '('a 'b))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('not_null)))) '('b (PgType 'text) '('columnConstraints '('('not_null)))))) '('primarykey '('a))))) (let world (CommitAll! world)) (return world) ) @@ -190,7 +190,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)) '('b (PgType 'text)))) '('primarykey '('a 'b)) '('notnull '('b 'a))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('not_null)))) '('b (PgType 'text) '('columnConstraints '('('not_null)))))) '('primarykey '('a 'b))))) (let world (CommitAll! world)) (return world) ) @@ -224,7 +224,7 @@ Y_UNIT_TEST_SUITE(PgSqlParsingOnly) { TString program = R"( ( (let world (Configure! world (DataSource 'config) 'OrderedColumns)) - (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4)))) '('serialColumns '('a))))) + (let world (Write! world (DataSink '"kikimr" '"") (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('a (PgType 'int4) '('columnConstraints '('('serial))))))))) (let world (CommitAll! world)) (return world)) )"; diff --git a/contrib/ydb/library/yql/sql/v1/query.cpp b/contrib/ydb/library/yql/sql/v1/query.cpp index 8ed02cf4519..24b01b27520 100644 --- a/contrib/ydb/library/yql/sql/v1/query.cpp +++ b/contrib/ydb/library/yql/sql/v1/query.cpp @@ -812,34 +812,29 @@ class TCreateTableNode final: public TAstListNode { opts = L(opts, Q(Y(Q("columnFamilies"), Q(columnFamilies)))); } - THashSet notNullColumnsSet; - auto notNullColumns = Y(); auto columns = Y(); - THashSet serialColumnsSet; THashSet columnsWithDefaultValue; auto columnsDefaultValueSettings = Y(); - auto serialColumns = Y(); for (auto& col : Params.Columns) { auto columnDesc = Y(); columnDesc = L(columnDesc, BuildQuotedAtom(Pos, col.Name)); auto type = col.Type; - if (col.Serial) { - if (serialColumnsSet.insert(col.Name).second) - serialColumns = L(serialColumns, BuildQuotedAtom(Pos, col.Name)); - } + if (col.Nullable) { type = Y("AsOptionalType", type); - } else { - if (notNullColumnsSet.insert(col.Name).second) - notNullColumns = L(notNullColumns, BuildQuotedAtom(Pos, col.Name)); } columnDesc = L(columnDesc, type); auto columnConstraints = Y(); + + if (!col.Nullable) { + columnConstraints = L(columnConstraints, Q(Y(Q("not_null")))); + } + if (col.Serial) { - columnConstraints = L(columnConstraints, Q(Y(Q("serial"), Q("true")))); + columnConstraints = L(columnConstraints, Q(Y(Q("serial")))); } if (col.DefaultExpr) { @@ -898,14 +893,6 @@ class TCreateTableNode final: public TAstListNode { } } - if (!notNullColumnsSet.empty()) { - opts = L(opts, Q(Y(Q("notnull"), Q(notNullColumns)))); - } - - if (!serialColumnsSet.empty()) { - opts = L(opts, Q(Y(Q("serialColumns"), Q(serialColumns)))); - } - if (!Params.PartitionByColumns.empty()) { auto partitionBy = Y(); for (auto& col : Params.PartitionByColumns) { @@ -1089,8 +1076,12 @@ class TAlterTableNode final: public TAstListNode { columnDesc = L(columnDesc, type); auto columnConstraints = Y(); + if (!col.Nullable) { + columnConstraints = L(columnConstraints, Q(Y(Q("not_null")))); + } + if (col.Serial) { - columnConstraints = L(columnConstraints, Q(Y(Q("serial"), Q("true")))); + columnConstraints = L(columnConstraints, Q(Y(Q("serial")))); } if (col.DefaultExpr) { diff --git a/contrib/ydb/library/yql/sql/v1/sql_ut.cpp b/contrib/ydb/library/yql/sql/v1/sql_ut.cpp index f04a00918fc..4d74be1b0fb 100644 --- a/contrib/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/contrib/ydb/library/yql/sql/v1/sql_ut.cpp @@ -828,7 +828,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write!") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '()) '()))) '('notnull '('"a")))))__")); + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '())))))))__")); } }; @@ -862,7 +862,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write!") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('"a" (PgType '_int4) '('columnConstrains '()) '()))) '('notnull '('"a")))))__")); + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('"a" (PgType '_int4) '('columnConstrains '('('not_null))) '())))))))__")); } }; @@ -915,7 +915,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) { if (word == "Write!") { UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, - line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '()) '()))) '('primarykey '('"a")) '('notnull '('"a")))))__")); + line.find(R"__((Write! world sink (Key '('tablescheme (String '"t"))) (Void) '('('mode 'create) '('columns '('('"a" (DataType 'Int32) '('columnConstrains '('('not_null))) '()))) '('primarykey '('"a"))))))__")); } }; diff --git a/contrib/ydb/library/yql/utils/CMakeLists.darwin-x86_64.txt b/contrib/ydb/library/yql/utils/CMakeLists.darwin-x86_64.txt index 8d4e3de3516..49d10ddfe61 100644 --- a/contrib/ydb/library/yql/utils/CMakeLists.darwin-x86_64.txt +++ b/contrib/ydb/library/yql/utils/CMakeLists.darwin-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(backtrace) add_subdirectory(failure_injector) add_subdirectory(fetch) add_subdirectory(log) +add_subdirectory(plan) add_subdirectory(threading) add_library(library-yql-utils) diff --git a/contrib/ydb/library/yql/utils/CMakeLists.linux-aarch64.txt b/contrib/ydb/library/yql/utils/CMakeLists.linux-aarch64.txt index 06215c76cfc..5b6404ec307 100644 --- a/contrib/ydb/library/yql/utils/CMakeLists.linux-aarch64.txt +++ b/contrib/ydb/library/yql/utils/CMakeLists.linux-aarch64.txt @@ -12,6 +12,7 @@ add_subdirectory(backtrace) add_subdirectory(failure_injector) add_subdirectory(fetch) add_subdirectory(log) +add_subdirectory(plan) add_subdirectory(threading) add_library(library-yql-utils) diff --git a/contrib/ydb/library/yql/utils/CMakeLists.linux-x86_64.txt b/contrib/ydb/library/yql/utils/CMakeLists.linux-x86_64.txt index 06215c76cfc..5b6404ec307 100644 --- a/contrib/ydb/library/yql/utils/CMakeLists.linux-x86_64.txt +++ b/contrib/ydb/library/yql/utils/CMakeLists.linux-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(backtrace) add_subdirectory(failure_injector) add_subdirectory(fetch) add_subdirectory(log) +add_subdirectory(plan) add_subdirectory(threading) add_library(library-yql-utils) diff --git a/contrib/ydb/library/yql/utils/CMakeLists.windows-x86_64.txt b/contrib/ydb/library/yql/utils/CMakeLists.windows-x86_64.txt index 8d4e3de3516..49d10ddfe61 100644 --- a/contrib/ydb/library/yql/utils/CMakeLists.windows-x86_64.txt +++ b/contrib/ydb/library/yql/utils/CMakeLists.windows-x86_64.txt @@ -12,6 +12,7 @@ add_subdirectory(backtrace) add_subdirectory(failure_injector) add_subdirectory(fetch) add_subdirectory(log) +add_subdirectory(plan) add_subdirectory(threading) add_library(library-yql-utils) diff --git a/contrib/ydb/library/yql/utils/plan/CMakeLists.darwin-x86_64.txt b/contrib/ydb/library/yql/utils/plan/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..20bd929b461 --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-utils-plan) +target_link_libraries(yql-utils-plan PUBLIC + contrib-libs-cxxsupp + yutil + library-yql-ast + yql-core-expr_nodes +) +target_sources(yql-utils-plan PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/utils/plan/plan_utils.cpp +) diff --git a/contrib/ydb/library/yql/utils/plan/CMakeLists.linux-aarch64.txt b/contrib/ydb/library/yql/utils/plan/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..3b91c0925df --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/CMakeLists.linux-aarch64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-utils-plan) +target_link_libraries(yql-utils-plan PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-ast + yql-core-expr_nodes +) +target_sources(yql-utils-plan PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/utils/plan/plan_utils.cpp +) diff --git a/contrib/ydb/library/yql/utils/plan/CMakeLists.linux-x86_64.txt b/contrib/ydb/library/yql/utils/plan/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..3b91c0925df --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/CMakeLists.linux-x86_64.txt @@ -0,0 +1,20 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-utils-plan) +target_link_libraries(yql-utils-plan PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-yql-ast + yql-core-expr_nodes +) +target_sources(yql-utils-plan PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/utils/plan/plan_utils.cpp +) diff --git a/contrib/ydb/library/yql/utils/plan/CMakeLists.txt b/contrib/ydb/library/yql/utils/plan/CMakeLists.txt new file mode 100644 index 00000000000..43b1a6b00e8 --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +endif() diff --git a/contrib/ydb/library/yql/utils/plan/CMakeLists.windows-x86_64.txt b/contrib/ydb/library/yql/utils/plan/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..20bd929b461 --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/CMakeLists.windows-x86_64.txt @@ -0,0 +1,19 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yql-utils-plan) +target_link_libraries(yql-utils-plan PUBLIC + contrib-libs-cxxsupp + yutil + library-yql-ast + yql-core-expr_nodes +) +target_sources(yql-utils-plan PRIVATE + ${CMAKE_SOURCE_DIR}/contrib/ydb/library/yql/utils/plan/plan_utils.cpp +) diff --git a/contrib/ydb/library/yql/utils/plan/plan_utils.cpp b/contrib/ydb/library/yql/utils/plan/plan_utils.cpp new file mode 100644 index 00000000000..1f6f9fe0eb6 --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/plan_utils.cpp @@ -0,0 +1,179 @@ +#include "plan_utils.h" + +#include + +#include + +#include + +namespace NYql::NPlanUtils { + +using namespace NNodes; + +TPredicate ExtractPredicate(const TCoLambda& expr) { + TPredicate pred; + pred.Args.reserve(expr.Args().Ref().ChildrenSize()); + for (const auto& child : expr.Args().Ref().Children()) { + pred.Args.push_back(PrettyExprStr(TExprBase(child))); + } + + pred.Body = PrettyExprStr(expr.Body()); + return pred; +} + +TString ToStr(const TCoDataCtor& data) { + TStringStream out; + EscapeArbitraryAtom(data.Literal().Value(), '"', &out); + return out.Str(); +} + +TString ToStr(const TCoLambda& lambda) { + return PrettyExprStr(lambda.Body()); +} + +TString ToStr(const TCoAsStruct& asStruct) { + TVector args; + for (const auto& kv : asStruct.Args()) { + auto key = PrettyExprStr(TExprBase(kv->Child(0))); + auto value = PrettyExprStr(TExprBase(kv->Child(1))); + + if (!key.empty() && !value.empty()) { + args.push_back(TStringBuilder() << key << ": " << value); + } + } + + return TStringBuilder() << "{" << JoinStrings(std::move(args), ",") << "}"; +} + +TString ToStr(const TCoAsList& asList) { + TVector args; + for (const auto& arg : asList.Args()) { + if (auto str = PrettyExprStr(TExprBase(arg))) { + args.push_back(std::move(str)); + } + } + + return TStringBuilder() << "[" << JoinStrings(std::move(args), ",") << "]"; +} + +TString ToStr(const TCoMember& member) { + auto structName = PrettyExprStr(member.Struct()); + auto memberName = PrettyExprStr(member.Name()); + + if (!structName.empty() && !memberName.empty()) { + return TStringBuilder() << structName << "." << memberName; + } + + return {}; +} + +TString ToStr(const TCoIfPresent& ifPresent) { + /* expected IfPresent with 3 children: + * 0-Optional, 1-PresentHandler, 2-MissingValue */ + if (ifPresent.Ref().ChildrenSize() == 3) { + auto arg = PrettyExprStr(ifPresent.Optional()); + auto pred = ExtractPredicate(ifPresent.PresentHandler()); + + Y_ENSURE(!pred.Args.empty()); + return std::regex_replace(pred.Body.c_str(), + std::regex(pred.Args[0].c_str()), arg.c_str()).data(); + } + + return "..."; +} + +TString ToStr(const TCoExists& exist) { + if (auto str = PrettyExprStr(exist.Optional())) { + return TStringBuilder() << "Exist(" << str << ")"; + } + + return {}; +} + +TString AggrOpToStr(const TExprBase& aggr) { + TVector args; + for (const auto& child : aggr.Ref().Children()) { + if (auto str = PrettyExprStr(TExprBase(child))) { + args.push_back(std::move(str)); + } + } + + return TStringBuilder() << aggr.Ref().Content() << "(" + << JoinStrings(std::move(args), ",") << ")"; +} + +TString BinaryOpToStr(const TExprBase& op) { + auto left = PrettyExprStr(TExprBase(op.Ref().Child(0))); + auto right = PrettyExprStr(TExprBase(op.Ref().Child(1))); + + TStringBuilder str; + str << left; + if (left && right) { + str << " " << op.Ref().Content() << " "; + } + str << right; + + return str; +} + +TString LogicOpToStr(const TExprBase& op) { + TVector args; + for (const auto& child : op.Ref().Children()) { + if (auto str = PrettyExprStr(TExprBase(child))) { + args.push_back(std::move(str)); + } + } + + return JoinStrings(std::move(args), TStringBuilder() << " " << op.Ref().Content() << " "); +} + +TString NotToStr(const TCoNot& notOp) { + return TStringBuilder() << "Not " << PrettyExprStr(notOp.Value()); +} + +TString PrettyExprStr(const TExprBase& expr) { + static const THashMap aggregations = { + {"AggrMin", "MIN"}, + {"AggrMax", "MAX"}, + {"AggrCountUpdate", "COUNT"}, + {"AggrAdd", "SUM"} + }; + + if (expr.Maybe()) { + return TString(expr.Ref().Child(0)->Content()); + } else if (auto data = expr.Maybe()) { + return ToStr(data.Cast()); + } else if (auto lambda = expr.Maybe()) { + return ToStr(lambda.Cast()); + } else if (auto asStruct = expr.Maybe()) { + return ToStr(asStruct.Cast()); + } else if (auto asList = expr.Maybe()) { + return ToStr(asList.Cast()); + } else if (auto member = expr.Maybe()) { + return ToStr(member.Cast()); + } else if (auto ifPresent = expr.Maybe()) { + return ToStr(ifPresent.Cast()); + } else if (auto exist = expr.Maybe()) { + return ToStr(exist.Cast()); + } else if (expr.Maybe() || expr.Maybe() || expr.Maybe()) { + return AggrOpToStr(expr); + } else if (aggregations.contains(expr.Ref().Content())) { + return TStringBuilder() << aggregations.at(expr.Ref().Content()) << "(" + << PrettyExprStr(TExprBase(expr.Ref().Child(0))) << ")"; + } else if (expr.Maybe() || expr.Maybe()) { + return BinaryOpToStr(expr); + } else if (expr.Maybe() || expr.Maybe() || expr.Maybe()) { + return LogicOpToStr(expr); + } else if (auto notOp = expr.Maybe()) { + return NotToStr(notOp.Cast()); + } else if (expr.Maybe() || expr.Maybe() || expr.Maybe() + || expr.Maybe() || expr.Maybe()) { + return PrettyExprStr(TExprBase(expr.Ref().Child(0))); + } else { + return TString(expr.Ref().Content()); + } + + return {}; +} + +} // namespace NYql::NPlanUtils diff --git a/contrib/ydb/library/yql/utils/plan/plan_utils.h b/contrib/ydb/library/yql/utils/plan/plan_utils.h new file mode 100644 index 00000000000..fd97328c257 --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/plan_utils.h @@ -0,0 +1,17 @@ +#pragma once +#include + +#include + +namespace NYql::NPlanUtils { + +struct TPredicate { + TVector Args; + TString Body; +}; + +TPredicate ExtractPredicate(const NNodes::TCoLambda& expr); + +TString PrettyExprStr(const NNodes::TExprBase& expr); + +} // namespace NYql::NPlanUtils diff --git a/contrib/ydb/library/yql/utils/plan/plan_utils_ut.cpp b/contrib/ydb/library/yql/utils/plan/plan_utils_ut.cpp new file mode 100644 index 00000000000..0c2c5a1f98a --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/plan_utils_ut.cpp @@ -0,0 +1,83 @@ +#include "plan_utils.h" + +#include +#include + +#include + +#include + +using namespace NYql; +using namespace NNodes; +using namespace NPlanUtils; + +TExprNode::TPtr ConstantExprNode(TExprContext& ctx, i64 c) { + auto pos = ctx.AppendPosition({}); + return ctx.Builder(pos).Callable("Int64").Atom(0, ToString(c)).Seal().Build(); +} + +TString PrettifyForPlan(TExprContext& ctx, const TExprBase& ex) { + Cerr << "Prettify node:\n" << NCommon::ExprToPrettyString(ctx, *ex.Raw()) << Endl; + TString result = PrettyExprStr(ex); + Cerr << "Result: [" << result << "]" << Endl; + return result; +} + +Y_UNIT_TEST_SUITE(PlanUtilsTest) { + Y_UNIT_TEST(Cmp) { + TExprContext ctx; + auto lambda = Build(ctx, ctx.AppendPosition({})) + .Args({"arg"}) + .Body() + .Left() + .Struct("arg") + .Name().Build("a") + .Build() + .Right(ConstantExprNode(ctx, 42)) + .Build() + .Done(); + UNIT_ASSERT_STRINGS_EQUAL(PrettifyForPlan(ctx, lambda), "arg.a > 42"); + } + + Y_UNIT_TEST(Exists) { + TExprContext ctx; + auto lambda = Build(ctx, ctx.AppendPosition({})) + .Args({"s"}) + .Body() + .Optional() + .Struct("s") + .Name().Build("m") + .Build() + .Build() + .Done(); + UNIT_ASSERT_STRINGS_EQUAL(PrettifyForPlan(ctx, lambda), "Exist(s.m)"); + } + + Y_UNIT_TEST(And) { + TExprContext ctx; + auto lambda = Build(ctx, ctx.AppendPosition({})) + .Args({"row"}) + .Body() + .Add() + .Value() + .Optional() + .Struct("row") + .Name().Build("x") + .Build() + .Build() + .Build() + .Add() + .Left() + .Struct("row") + .Name().Build("y") + .Build() + .Right() + .Struct("row") + .Name().Build("z") + .Build() + .Build() + .Build() + .Done(); + UNIT_ASSERT_STRINGS_EQUAL(PrettifyForPlan(ctx, lambda), "Not Exist(row.x) And row.y == row.z"); + } +} diff --git a/contrib/ydb/library/yql/utils/plan/ut/ya.make b/contrib/ydb/library/yql/utils/plan/ut/ya.make new file mode 100644 index 00000000000..ec677de3c3f --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/ut/ya.make @@ -0,0 +1,16 @@ +UNITTEST_FOR(contrib/ydb/library/yql/utils/plan) + +SRCS( + plan_utils_ut.cpp +) + +PEERDIR( + contrib/ydb/library/yql/ast + contrib/ydb/library/yql/providers/common/provider + contrib/ydb/library/yql/public/udf/service/stub + contrib/ydb/library/yql/sql/pg_dummy +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/contrib/ydb/library/yql/utils/plan/ya.make b/contrib/ydb/library/yql/utils/plan/ya.make new file mode 100644 index 00000000000..8cfa6a2d038 --- /dev/null +++ b/contrib/ydb/library/yql/utils/plan/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +PEERDIR( + contrib/ydb/library/yql/ast + contrib/ydb/library/yql/core/expr_nodes +) + +SRCS( + plan_utils.cpp +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/contrib/ydb/library/yql/utils/ya.make b/contrib/ydb/library/yql/utils/ya.make index 0af4fd42a6b..bf7ee647552 100644 --- a/contrib/ydb/library/yql/utils/ya.make +++ b/contrib/ydb/library/yql/utils/ya.make @@ -60,6 +60,7 @@ RECURSE( failure_injector fetch log + plan simd sys test_http_server diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp b/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp deleted file mode 100644 index cf5d01ef702..00000000000 --- a/library/cpp/yaml/fyamlcpp/fyamlcpp.cpp +++ /dev/null @@ -1,1126 +0,0 @@ -#include "fyamlcpp.h" - -#include - -#include - -namespace NFyaml { - -const char* zstr = ""; - -enum class EErrorType { - Debug = FYET_DEBUG, - Info = FYET_INFO, - Notice = FYET_NOTICE, - Warning = FYET_WARNING, - Error = FYET_ERROR, - Max = FYET_MAX, -}; - -enum class EErrorModule { - Unknown = FYEM_UNKNOWN, - Atom = FYEM_ATOM, - Scan = FYEM_SCAN, - Parse = FYEM_PARSE, - Doc = FYEM_DOC, - Build = FYEM_BUILD, - Internal = FYEM_INTERNAL, - System = FYEM_SYSTEM, - Max = FYEM_MAX, -}; - -enum class EParseCfgFlags { - Quiet = FYPCF_QUIET, - CollectDiag = FYPCF_COLLECT_DIAG, - ResolveDocument = FYPCF_RESOLVE_DOCUMENT, - DisableMmapOpt = FYPCF_DISABLE_MMAP_OPT, - DisableRecycling = FYPCF_DISABLE_RECYCLING, - ParseComments = FYPCF_PARSE_COMMENTS, - DisableDepth_limit = FYPCF_DISABLE_DEPTH_LIMIT, - DisableAccelerators = FYPCF_DISABLE_ACCELERATORS, - DisableBuffering = FYPCF_DISABLE_BUFFERING, - DefaultVersionAuto = FYPCF_DEFAULT_VERSION_AUTO, - DefaultVersion1_1 = FYPCF_DEFAULT_VERSION_1_1, - DefaultVersion1_2 = FYPCF_DEFAULT_VERSION_1_2, - DefaultVersion1_3 = FYPCF_DEFAULT_VERSION_1_3, - SloppyFlowIndentation = FYPCF_SLOPPY_FLOW_INDENTATION, - PreferRecursive = FYPCF_PREFER_RECURSIVE, - JsonAuto = FYPCF_JSON_AUTO, - JsonNone = FYPCF_JSON_NONE, - JsonForce = FYPCF_JSON_FORCE, - YpathAliases = FYPCF_YPATH_ALIASES, - AllowDuplicateKeys = FYPCF_ALLOW_DUPLICATE_KEYS, -}; - -enum class EEventType { - None = FYET_NONE, - StreamStart = FYET_STREAM_START, - StreamEnd = FYET_STREAM_END, - DocumentStart = FYET_DOCUMENT_START, - DocumentEnd = FYET_DOCUMENT_END, - MappingStart = FYET_MAPPING_START, - MappingEnd = FYET_MAPPING_END, - SequenceStart = FYET_SEQUENCE_START, - SequenceEnd = FYET_SEQUENCE_END, - Scalar = FYET_SCALAR, - Alias = FYET_ALIAS, -}; - -enum class EScalarStyle { - Any = FYSS_ANY, - Plain = FYSS_PLAIN, - SingleQuoted = FYSS_SINGLE_QUOTED, - DoubleQuoted = FYSS_DOUBLE_QUOTED, - Literal = FYSS_LITERAL, - Folded = FYSS_FOLDED, - Max = FYSS_MAX, -}; - -enum class EEmitterWriteType { - DocumentIndicator = fyewt_document_indicator, - TagDirective = fyewt_tag_directive, - VersionDirective = fyewt_version_directive, - Indent = fyewt_indent, - Indicator = fyewt_indicator, - Whitespace = fyewt_whitespace, - PlainScalar = fyewt_plain_scalar, - SingleQuotedScalar = fyewt_single_quoted_scalar, - DoubleQuotedScalar = fyewt_double_quoted_scalar, - LiteralScalar = fyewt_literal_scalar, - FoldedScalar = fyewt_folded_scalar, - Anchor = fyewt_anchor, - Tag = fyewt_tag, - Linebreak = fyewt_linebreak, - Alias = fyewt_alias, - TerminatingZero = fyewt_terminating_zero, - PlainScalarKey = fyewt_plain_scalar_key, - SingleQuotedScalarKey = fyewt_single_quoted_scalar_key, - DoubleQuotedScalarKey = fyewt_double_quoted_scalar_key, - Comment = fyewt_comment, -}; - -enum class ECommentPlacement { - Top = fycp_top, - Right = fycp_right, - Bottom = fycp_bottom, -}; - -enum EEmitterCfgFlags { - SortKeys = FYECF_SORT_KEYS, - OutputComments = FYECF_OUTPUT_COMMENTS, - StripLabels = FYECF_STRIP_LABELS, - StripTags = FYECF_STRIP_TAGS, - StripDoc = FYECF_STRIP_DOC, - NoEndingNewline = FYECF_NO_ENDING_NEWLINE, - StripEmptyKv = FYECF_STRIP_EMPTY_KV, - IndentDefault = FYECF_INDENT_DEFAULT, - Indent1 = FYECF_INDENT_1, - Indent2 = FYECF_INDENT_2, - Indent3 = FYECF_INDENT_3, - Indent4 = FYECF_INDENT_4, - Indent5 = FYECF_INDENT_5, - Indent6 = FYECF_INDENT_6, - Indent7 = FYECF_INDENT_7, - Indent8 = FYECF_INDENT_8, - Indent9 = FYECF_INDENT_9, - WidthDefault = FYECF_WIDTH_DEFAULT, - Width80 = FYECF_WIDTH_80, - Width132 = FYECF_WIDTH_132, - WidthInf = FYECF_WIDTH_INF, - ModeOriginal = FYECF_MODE_ORIGINAL, - ModeBlock = FYECF_MODE_BLOCK, - ModeFlow = FYECF_MODE_FLOW, - ModeFlowOneline = FYECF_MODE_FLOW_ONELINE, - ModeJson = FYECF_MODE_JSON, - ModeJsonTp = FYECF_MODE_JSON_TP, - ModeJsonOneline = FYECF_MODE_JSON_ONELINE, - ModeDejson = FYECF_MODE_DEJSON, - ModePretty = FYECF_MODE_PRETTY, - DocStartMarkAuto = FYECF_DOC_START_MARK_AUTO, - DocStartMarkOff = FYECF_DOC_START_MARK_OFF, - DocStartMarkOn = FYECF_DOC_START_MARK_ON, - DocEndMarkAuto = FYECF_DOC_END_MARK_AUTO, - DocEndMarkOff = FYECF_DOC_END_MARK_OFF, - DocEndMarkOn = FYECF_DOC_END_MARK_ON, - VersionDirAuto = FYECF_VERSION_DIR_AUTO, - VersionDirOff = FYECF_VERSION_DIR_OFF, - VersionDirOn = FYECF_VERSION_DIR_ON, - TagDirAuto = FYECF_TAG_DIR_AUTO, - TagDirOff = FYECF_TAG_DIR_OFF, - TagDirOn = FYECF_TAG_DIR_ON, - - Default = FYECF_DEFAULT, -}; - -enum class ENodeWalkFlags { - DontFollow = FYNWF_DONT_FOLLOW, - Follow = FYNWF_FOLLOW, - PtrYaml = FYNWF_PTR_YAML, - PtrJson = FYNWF_PTR_JSON, - PtrReljson = FYNWF_PTR_RELJSON, - PtrYpath = FYNWF_PTR_YPATH, - UriEncoded = FYNWF_URI_ENCODED, - MaxdepthDefault = FYNWF_MAXDEPTH_DEFAULT, - MarkerDefault = FYNWF_MARKER_DEFAULT, - PtrDefault = FYNWF_PTR_DEFAULT, -}; - -enum class EPathParseCfgFlags { - Quiet = FYPPCF_QUIET, - DisableRecycling = FYPPCF_DISABLE_RECYCLING, - DisableAccelerators = FYPPCF_DISABLE_ACCELERATORS, -}; - -enum class EPathExecCfgFlags { - Quiet = FYPXCF_QUIET, - DisableRecycling = FYPXCF_DISABLE_RECYCLING, - DisableAccelerators = FYPXCF_DISABLE_ACCELERATORS, -}; - -enum class ETokenType { - /* non-content token types */ - None = FYTT_NONE, - StreamStart = FYTT_STREAM_START, - StreamEnd = FYTT_STREAM_END, - VersionDirective = FYTT_VERSION_DIRECTIVE, - TagDirective = FYTT_TAG_DIRECTIVE, - DocumentStart = FYTT_DOCUMENT_START, - DocumentEnd = FYTT_DOCUMENT_END, - /* content token types */ - BlockSequenceStart = FYTT_BLOCK_SEQUENCE_START, - BlockMappingStart = FYTT_BLOCK_MAPPING_START, - BlockEnd = FYTT_BLOCK_END, - FlowSequenceStart = FYTT_FLOW_SEQUENCE_START, - FlowSequenceEnd = FYTT_FLOW_SEQUENCE_END, - FlowMappingStart = FYTT_FLOW_MAPPING_START, - FlowMappingEnd = FYTT_FLOW_MAPPING_END, - BlockEntry = FYTT_BLOCK_ENTRY, - FlowEntry = FYTT_FLOW_ENTRY, - Key = FYTT_KEY, - Value = FYTT_VALUE, - Alias = FYTT_ALIAS, - Anchor = FYTT_ANCHOR, - Tag = FYTT_TAG, - Scalar = FYTT_SCALAR, - - /* special error reporting */ - Input_marker = FYTT_INPUT_MARKER, - - /* path expression tokens */ - PeSlash = FYTT_PE_SLASH, - PeRoot = FYTT_PE_ROOT, - PeThis = FYTT_PE_THIS, - PeParent = FYTT_PE_PARENT, - PeMapKey = FYTT_PE_MAP_KEY, - PeSeqIndex = FYTT_PE_SEQ_INDEX, - PeSeqSlice = FYTT_PE_SEQ_SLICE, - PeScalarFilter = FYTT_PE_SCALAR_FILTER, - PeCollectionFilter = FYTT_PE_COLLECTION_FILTER, - PeSeqFilter = FYTT_PE_SEQ_FILTER, - PeMapFilter = FYTT_PE_MAP_FILTER, - PeUniqueFilter = FYTT_PE_UNIQUE_FILTER, - PeEveryChild = FYTT_PE_EVERY_CHILD, - PeEveryChildR = FYTT_PE_EVERY_CHILD_R, - PeAlias = FYTT_PE_ALIAS, - PeSibling = FYTT_PE_SIBLING, - PeComma = FYTT_PE_COMMA, - PeBarbar = FYTT_PE_BARBAR, - PeAmpamp = FYTT_PE_AMPAMP, - PeLparen = FYTT_PE_LPAREN, - PeRparen = FYTT_PE_RPAREN, - - /* comparison operators */ - PeEqeq = FYTT_PE_EQEQ, - PeNoteq = FYTT_PE_NOTEQ, - PeLt = FYTT_PE_LT, - PeGt = FYTT_PE_GT, - PeLte = FYTT_PE_LTE, - PeGte = FYTT_PE_GTE, - - /* scalar expression tokens */ - SePlus = FYTT_SE_PLUS, - SeMinus = FYTT_SE_MINUS, - SeMult = FYTT_SE_MULT, - SeDiv = FYTT_SE_DIV, - - PeMethod = FYTT_PE_METHOD, - SeMethod = FYTT_SE_METHOD, -}; - -enum class EComposerReturn { - OkContinue = FYCR_OK_CONTINUE, - OkStop = FYCR_OK_STOP, - OkStartSkip = FYCR_OK_START_SKIP, - OkStopSkip = FYCR_OK_STOP_SKIP, - Error = FYCR_ERROR, -}; - -TDocumentIterator::TDocumentIterator(fy_document_iterator* iterator) - : Iterator_(iterator, fy_document_iterator_destroy) -{} - -TNodeRef::TNodeRef(fy_node* node) - : Node_(node) -{} - -fy_node* TNodeRef::NodeRawPointer() const { - return Node_; -} - -TNode& TNode::operator=(fy_node* node) { - Node_.reset(node, fy_node_free); - return *this; -} - -TNode::TNode(fy_node* node) - : Node_(node, fy_node_free) -{} - -TNodeRef TNodePairRef::Key() const { - ENSURE_NODE_NOT_EMPTY(Pair_); - return TNodeRef(fy_node_pair_key(Pair_)); -} - -int TNodePairRef::Index(const TNodeRef& node) const { - ENSURE_NODE_NOT_EMPTY(node); - ENSURE_NODE_NOT_EMPTY(Pair_); - return fy_node_mapping_get_pair_index(node.Node_, Pair_); -} - -void TNodePairRef::SetKey(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Pair_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_pair_set_key(Pair_, node.Node_), Pair_); -} - -TNodeRef TNodePairRef::Value() const { - ENSURE_NODE_NOT_EMPTY(Pair_); - return TNodeRef(fy_node_pair_value(Pair_)); -} - -void TNodePairRef::SetValue(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Pair_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_pair_set_value(Pair_, node.Node_), Pair_); -} - -TMappingIterator::TMappingIterator(const TNodeRef& node, bool end) - : Node_(node) -{ - if (!end) { - NodePair_ = TNodePairRef(fy_node_mapping_iterate(Node_.Node_, reinterpret_cast(&NodePair_.Pair_))); - } -} - -TMappingIterator& TMappingIterator::operator++() { - NodePair_ = TNodePairRef(fy_node_mapping_iterate(Node_.Node_, reinterpret_cast(&NodePair_.Pair_))); - return *this; -} - -TReverseMappingIterator::TReverseMappingIterator(const TNodeRef& node, bool end) - : Node_(node) -{ - if (!end) { - NodePair_ = TNodePairRef(fy_node_mapping_reverse_iterate(Node_.Node_, reinterpret_cast(&NodePair_.Pair_))); - } -} - -TReverseMappingIterator& TReverseMappingIterator::operator++() { - NodePair_ = TNodePairRef(fy_node_mapping_reverse_iterate(Node_.Node_, reinterpret_cast(&NodePair_.Pair_))); - return *this; -} - -size_t TMapping::size() const { - ENSURE_NODE_NOT_EMPTY(Node_); - return fy_node_mapping_item_count(Node_); -} - -size_t TMapping::empty() const { - ENSURE_NODE_NOT_EMPTY(Node_); - return fy_node_mapping_is_empty(Node_); -} - -TNodePairRef TMapping::at(int index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - auto res = fy_node_mapping_get_by_index(Node_, index); - Y_ENSURE_EX(res, ({ - TStringStream ss; - ss << "No such child: " << Path() << "/" << index; - TFyamlEx(ss.Str()); - })); - return TNodePairRef(res); -} - -TNodePairRef TMapping::operator[](int index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - return TNodePairRef(fy_node_mapping_get_by_index(Node_, index)); -} - -TNodeRef TMapping::at(const TString& index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - auto res = fy_node_mapping_lookup_by_string(Node_, index.data(), index.size()); - Y_ENSURE_EX(res, ({ - TStringStream ss; - ss << "No such child: " << Path() << "/" << index; - TFyamlEx(ss.Str()); - })); - return TNodeRef(res); -} - -TNodePairRef TMapping::pair_at(const TString& index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - auto res = fy_node_mapping_lookup_pair_by_string(Node_, index.data(), index.size()); - Y_ENSURE_EX(res, ({ - TStringStream ss; - ss << "No such child: " << Path() << "/" << index; - TFyamlEx(ss.Str()); - })); - return TNodePairRef(res); -} - -TNodePairRef TMapping::pair_at_opt(const TString& index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - return TNodePairRef(fy_node_mapping_lookup_pair_by_string(Node_, index.data(), index.size())); -} - -TNodeRef TMapping::operator[](const TString& index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - return TNodeRef(fy_node_mapping_lookup_by_string(Node_, index.data(), index.size())); -} - -TNodeRef TMapping::operator[](const char* str) const { - ENSURE_NODE_NOT_EMPTY(Node_); - TString index(str); - return TNodeRef(fy_node_mapping_lookup_by_string(Node_, index.data(), index.size())); -} - -void TMapping::Append(const TNodeRef& key, const TNodeRef& value) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(key); - ENSURE_NODE_NOT_EMPTY(value); - NDetail::RethrowOnError(fy_node_mapping_append(Node_, key.Node_, value.Node_), Node_); -} - -void TMapping::Prepend(const TNodeRef& key, const TNodeRef& value) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(key); - ENSURE_NODE_NOT_EMPTY(value); - NDetail::RethrowOnError(fy_node_mapping_prepend(Node_, key.Node_, value.Node_), Node_); -} - -void TMapping::Remove(const TNodePairRef& toRemove) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(toRemove); - NDetail::RethrowOnError(fy_node_mapping_remove(Node_, toRemove.Pair_), Node_); - fy_node_free(fy_node_pair_key(toRemove.Pair_)); - fy_node_free(fy_node_pair_value(toRemove.Pair_)); - free(toRemove.Pair_); -} - -bool TMapping::Has(TString key) const { - return fy_node_mapping_lookup_by_string(Node_, key.data(), key.size()) != nullptr; -} - -TMappingIterator TMapping::Remove(const TMappingIterator& toRemove) { - ENSURE_NODE_NOT_EMPTY(Node_); - Y_DEBUG_ABORT_UNLESS(Node_ == toRemove.Node_); - TMappingIterator ret = toRemove; - ++ret; - fy_node_mapping_remove(Node_, toRemove.NodePair_.Pair_); - return ret; -} - -void TMapping::Remove(const TNodeRef& key) { - ENSURE_NODE_NOT_EMPTY(Node_); - fy_node_free(fy_node_mapping_remove_by_key(Node_, key.Node_)); -} - -TSequenceIterator::TSequenceIterator(const TNodeRef& node, bool end) - : Node_(node) -{ - if (!end) { - IterNode_ = TNodeRef(fy_node_sequence_iterate(Node_.Node_, &Iter_)); - } -} - -TSequenceIterator& TSequenceIterator::operator++() { - IterNode_ = TNodeRef(fy_node_sequence_iterate(Node_.Node_, &Iter_)); - return *this; -} - -void TSequenceIterator::InsertBefore(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(IterNode_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_insert_before(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); -} - -void TSequenceIterator::InsertAfter(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(IterNode_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_insert_after(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); -} - -TReverseSequenceIterator::TReverseSequenceIterator(const TNodeRef& node, bool end) - : Node_(node) -{ - if (!end) { - IterNode_ = TNodeRef(fy_node_sequence_reverse_iterate(Node_.Node_, &Iter_)); - } -} - -TReverseSequenceIterator& TReverseSequenceIterator::operator++() { - IterNode_ = TNodeRef(fy_node_sequence_reverse_iterate(Node_.Node_, &Iter_)); - return *this; -} - -void TReverseSequenceIterator::InsertBefore(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(IterNode_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_insert_after(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); -} - -void TReverseSequenceIterator::InsertAfter(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(IterNode_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_insert_before(Node_.Node_, IterNode_.Node_, node.Node_), Node_.Node_); -} - -size_t TSequence::size() const { - ENSURE_NODE_NOT_EMPTY(Node_); - return fy_node_sequence_item_count(Node_); -} - -size_t TSequence::empty() const { - ENSURE_NODE_NOT_EMPTY(Node_); - return fy_node_sequence_is_empty(Node_); -} - -TNodeRef TSequence::at(int index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - auto res = fy_node_sequence_get_by_index(Node_, index); - Y_ENSURE_EX(res, ({ - TStringStream ss; - ss << "No such index: " << Path() << "/" << index; - TFyamlEx(ss.Str()); - })); - return TNodeRef(res); -} - -TNodeRef TSequence::operator[](int index) const { - ENSURE_NODE_NOT_EMPTY(Node_); - return TNodeRef(fy_node_sequence_get_by_index(Node_, index)); -} - -void TSequence::Append(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_append(Node_, node.Node_), Node_); -} - -void TSequence::Prepend(const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_prepend(Node_, node.Node_), Node_); -} - -void TSequence::InsertBefore(const TNodeRef& mark, const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(mark); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_insert_before(Node_, mark.Node_, node.Node_), Node_); -} - -void TSequence::InsertAfter(const TNodeRef& mark, const TNodeRef& node) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(mark); - ENSURE_NODE_NOT_EMPTY(node); - NDetail::RethrowOnError(fy_node_sequence_insert_after(Node_, mark.Node_, node.Node_), Node_); -} - -TNode TSequence::Remove(const TNodeRef& toRemove) { - ENSURE_NODE_NOT_EMPTY(Node_); - ENSURE_NODE_NOT_EMPTY(toRemove.Node_); - return TNode(fy_node_sequence_remove(Node_, toRemove.Node_)); -} - -TSequenceIterator TSequence::Remove(const TSequenceIterator& toRemove) { - ENSURE_NODE_NOT_EMPTY(Node_); - Y_DEBUG_ABORT_UNLESS(Node_ == toRemove.Node_); - ENSURE_NODE_NOT_EMPTY(toRemove.IterNode_); - TSequenceIterator ret = toRemove; - ++ret; - fy_node_sequence_remove(Node_, toRemove.IterNode_.Node_); - fy_node_free(toRemove.IterNode_.Node_); // TODO add extract - return ret; -} - -TReverseSequenceIterator TSequence::Remove(const TReverseSequenceIterator& toRemove) { - ENSURE_NODE_NOT_EMPTY(Node_); - Y_DEBUG_ABORT_UNLESS(Node_ == toRemove.Node_); - ENSURE_NODE_NOT_EMPTY(toRemove.IterNode_); - TReverseSequenceIterator ret = toRemove; - ++ret; - fy_node_sequence_remove(Node_, toRemove.IterNode_.Node_); - fy_node_free(toRemove.IterNode_.Node_); // TODO add extract - return ret; -} - -TDocumentNodeIterator::TDocumentNodeIterator(TNodeRef&& node) - : Node_(node) -{ - if (node) { - Iterator_ = {fy_document_iterator_create(), fy_document_iterator_destroy}; - fy_document_iterator_node_start(Iterator_.get(), node.Node_); - } -} - -TDocumentNodeIterator& TDocumentNodeIterator::operator++() { - Node_ = fy_document_iterator_node_next(Iterator_.get()); - return *this; -} - -TDocument::TDocument(TString str, fy_document* doc, fy_diag* diag) - : Document_(doc, fy_document_destroy) - , Diag_(diag, fy_diag_destroy) -{ - auto* userdata = new THashSet, TStringPtrHashT>({MakeSimpleShared(std::move(str))}); - fy_document_set_userdata(doc, userdata); - fy_document_register_on_destroy(doc, &DestroyDocumentStrings); - RegisterUserDataCleanup(); -} - -TDocument::TDocument(fy_document* doc, fy_diag* diag) - : Document_(doc, fy_document_destroy) - , Diag_(diag, fy_diag_destroy) -{ - RegisterUserDataCleanup(); -} - -TDocument TDocument::Parse(TString str) { - const char* cstr = str.empty() ? zstr : str.cbegin(); - fy_diag_cfg dcfg; - fy_diag_cfg_default(&dcfg); - std::unique_ptr diag(fy_diag_create(&dcfg), fy_diag_destroy); - fy_diag_set_collect_errors(diag.get(), true); - fy_parse_cfg cfg{ - "", - // FYPCF_PARSE_COMMENTS, - FYPCF_QUIET, - nullptr, - diag.get() - }; - fy_document* doc = fy_document_build_from_string(&cfg, cstr, FY_NT); - if (!doc) { - NDetail::ThrowAllExceptionsIfAny(diag.get()); - } - return TDocument(std::move(str), doc, diag.release()); -} - -TDocument TDocument::Clone() const { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - fy_document* doc = fy_document_clone(Document_.get()); - fy_document_set_userdata( - doc, - new THashSet, TStringPtrHashT>( - *reinterpret_cast, TStringPtrHashT>*>(fy_document_get_userdata(Document_.get())) - ) - ); - fy_document_register_on_destroy(doc, &DestroyDocumentStrings); - return TDocument(doc, fy_document_get_diag(doc)); -} - -void TDocument::InsertAt(const char* path, const TNodeRef& node) { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - NDetail::RethrowOnError(fy_document_insert_at(Document_.get(), path, FY_NT, node.Node_), Diag_.get()); -} - -TNodeRef TDocument::Buildf(const char* content) { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return TNodeRef(fy_node_build_from_string(Document_.get(), content, strlen(content))); -} - -void TDocument::Resolve() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - if (fy_document_resolve(Document_.get()) != 0) { - NDetail::ThrowAllExceptionsIfAny(Diag_.get()); - } -} - -bool TDocument::HasDirectives() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return fy_document_has_directives(Document_.get()); -} - -bool TDocument::HasExplicitDocumentStart() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return fy_document_has_explicit_document_start(Document_.get()); -} - -bool TDocument::HasExplicitDocumentEnd() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return fy_document_has_explicit_document_end(Document_.get()); -} - -void TDocument::SetParent(const TDocument& doc) { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - ENSURE_DOCUMENT_NOT_EMPTY(doc.Document_); - NDetail::RethrowOnError(fy_document_set_parent(doc.Document_.get(), Document_.release()), Diag_.get()); -} - -TNodeRef TDocument::Root() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return TNodeRef(fy_document_root(Document_.get())); -} - -void TDocument::SetRoot(const TNodeRef& node) { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - ENSURE_NODE_NOT_EMPTY(node.Node_); - NDetail::RethrowOnError(fy_document_set_root(Document_.get(), node.Node_), Diag_.get()); -} - -TNodeRef TDocument::CreateAlias(const TString& name) { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return TNodeRef(fy_node_create_alias_copy(Document_.get(), name.c_str(), name.length())); -} - -std::unique_ptr TDocument::EmitToCharArray() const { - std::unique_ptr res( - fy_emit_document_to_string( - Document_.get(), - (fy_emitter_cfg_flags)(FYECF_DEFAULT | FYECF_MODE_PRETTY | FYECF_OUTPUT_COMMENTS)), &NDetail::FreeChar); - return res; -} - -bool TDocument::RegisterUserDataCleanup() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - return fy_document_register_meta(Document_.get(), &DestroyUserData, nullptr) == 0; -} - -void TDocument::UnregisterUserDataCleanup() { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - fy_document_unregister_meta(Document_.get()); -} - -TMark TDocument::BeginMark() const { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - auto* fyds = fy_document_get_document_state(Document_.get()); - auto* mark = fy_document_state_start_mark(fyds); - return TMark{ - mark->input_pos, - mark->line, - mark->column, - }; -} - -TMark TDocument::EndMark() const { - ENSURE_DOCUMENT_NOT_EMPTY(Document_); - auto* fyds = fy_document_get_document_state(Document_.get()); - auto* mark = fy_document_state_end_mark(fyds); - return TMark{ - mark->input_pos, - mark->line, - mark->column, - }; -} - -std::unique_ptr TJsonEmitter::EmitToCharArray() const { - std::unique_ptr res( - fy_emit_node_to_string( - Node_.Node_, - (fy_emitter_cfg_flags)(FYECF_DEFAULT | FYECF_SORT_KEYS | FYECF_MODE_JSON_TP)), &NDetail::FreeChar); - return res; -} - -TParser::TParser(TString rawStream, fy_parser* parser, fy_diag* diag) - : RawDocumentStream_(std::move(rawStream)) - , Parser_(parser, fy_parser_destroy) - , Diag_(diag, fy_diag_destroy) -{} - -TParser TParser::Create(TString str) -{ - const char* stream = str.empty() ? zstr : str.cbegin(); - fy_diag_cfg dcfg; - fy_diag_cfg_default(&dcfg); - std::unique_ptr diag(fy_diag_create(&dcfg), fy_diag_destroy); - fy_diag_set_collect_errors(diag.get(), true); - fy_parse_cfg cfg{ - "", - // FYPCF_PARSE_COMMENTS, - FYPCF_QUIET, - nullptr, - diag.get() - }; - auto* parser = fy_parser_create(&cfg); - if (!parser) { - NDetail::ThrowAllExceptionsIfAny(diag.get()); - } - - fy_parser_set_string(parser, stream, -1); - - return TParser(std::move(str), parser, diag.release()); -} - -std::optional TParser::NextDocument() { - auto* doc = fy_parse_load_document(Parser_.get()); - if (!doc) { - return std::nullopt; - } - - return TDocument(RawDocumentStream_, doc, fy_document_get_diag(doc)); -} - -namespace NDetail { - -fy_node* TNodeOpsBase::CreateReference(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return fy_node_create_reference(node); -} - -fy_node* TNodeOpsBase::Copy(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return fy_node_copy(fy_node_document(node), node); -} - -fy_node* TNodeOpsBase::Copy(fy_node* node, fy_document* to) const { - ENSURE_NODE_NOT_EMPTY(node); - auto* fromDoc = fy_node_document(node); - auto& fromUserdata = *reinterpret_cast, TStringPtrHashT>*>(fy_document_get_userdata(fromDoc)); - auto& toUserdata = *reinterpret_cast, TStringPtrHashT>*>(fy_document_get_userdata(to)); - toUserdata.insert(fromUserdata.begin(), fromUserdata.end()); - return fy_node_copy(to, node); -} - -TString TNodeOpsBase::Path(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - char* path = fy_node_get_path(node); - - if (path) { - TString str(path); - free(path); - return str; - } - - return {}; -} - -ENodeType TNodeOpsBase::Type(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return static_cast(fy_node_get_type(node)); -} - -bool TNodeOpsBase::IsAlias(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return fy_node_is_alias(node); -} - -fy_node* TNodeOpsBase::ResolveAlias(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - Y_DEBUG_ABORT_UNLESS(IsAlias(node)); - return fy_node_resolve_alias(node); -} - -TString TNodeOpsBase::Scalar(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - Y_ENSURE_EX(fy_node_is_scalar(node), TFyamlEx() << "Node is not Scalar: " << Path(node)); - size_t size; - const char* text = fy_node_get_scalar(node, &size); - return TString(text, size); -} - -TMark TNodeOpsBase::BeginMark(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - std::unique_ptr it( - fy_document_iterator_create(), - &fy_document_iterator_destroy); - fy_document_iterator_node_start(it.get(), node); - auto deleter = [&](fy_event* fye){ fy_document_iterator_event_free(it.get(), fye); }; - std::unique_ptr ev( - fy_document_iterator_body_next(it.get()), - deleter); - auto* mark = fy_event_start_mark(ev.get()); - - if (!mark) { - ythrow yexception() << "can't get begin mark for a node"; - } - - return TMark{ - mark->input_pos, - mark->line, - mark->column, - }; -} - -namespace { - -fy_event_type GetOpenEventType(ENodeType type) { - switch(type) { - case ENodeType::Mapping: - return FYET_MAPPING_START; - case ENodeType::Sequence: - return FYET_SEQUENCE_START; - default: - Y_ABORT("Not a brackets type"); - } -} - -fy_event_type GetCloseEventType(ENodeType type) { - switch(type) { - case ENodeType::Mapping: - return FYET_MAPPING_END; - case ENodeType::Sequence: - return FYET_SEQUENCE_END; - default: - Y_ABORT("Not a brackets type"); - } -} - -} // anonymous namespace - -TMark TNodeOpsBase::EndMark(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - std::unique_ptr it( - fy_document_iterator_create(), - &fy_document_iterator_destroy); - fy_document_iterator_node_start(it.get(), node); - - auto deleter = [&](fy_event* fye){ fy_document_iterator_event_free(it.get(), fye); }; - std::unique_ptr prevEv( - nullptr, - deleter); - std::unique_ptr ev( - fy_document_iterator_body_next(it.get()), - deleter); - - if (IsComplexType(Type(node))) { - int openBrackets = 0; - if (ev->type == GetOpenEventType(Type(node))) { - ++openBrackets; - } - if (ev->type == GetCloseEventType(Type(node))) { - --openBrackets; - } - while (ev->type != GetCloseEventType(Type(node)) || openBrackets != 0) { - std::unique_ptr cur( - fy_document_iterator_body_next(it.get()), - deleter); - if (cur == nullptr) { - break; - } - if (cur->type == GetOpenEventType(Type(node))) { - ++openBrackets; - } - if (cur->type == GetCloseEventType(Type(node))) { - --openBrackets; - } - if (fy_event_get_node_style(cur.get()) != FYNS_BLOCK) { - prevEv.reset(ev.release()); - ev.reset(cur.release()); - } - } - } - - auto* mark = fy_event_end_mark(ev.get()); - - if (!mark && prevEv) { - mark = fy_event_end_mark(prevEv.get()); - } - - if (!mark) { - ythrow yexception() << "can't get end mark for a node"; - } - - return TMark{ - mark->input_pos, - mark->line, - mark->column, - }; -} - -fy_node* TNodeOpsBase::Map(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - Y_ENSURE_EX(fy_node_is_mapping(node), TFyamlEx() << "Node is not Mapping: " << Path(node)); - return node; -} - -fy_node* TNodeOpsBase::Sequence(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - Y_ENSURE_EX(fy_node_is_sequence(node), TFyamlEx() << "Node is not Sequence: " << Path(node)); - return node; -} - -void TNodeOpsBase::Insert(fy_node* thisNode, fy_node* node) { - ENSURE_NODE_NOT_EMPTY(node); - RethrowOnError(fy_node_insert(thisNode, node), thisNode); -} - -std::optional TNodeOpsBase::Tag(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - size_t len = 0; - const char* tag = fy_node_get_tag(node, &len); - - if (tag) { - return TString(tag, len); - } - - return std::nullopt; -} - -void TNodeOpsBase::SetTag(fy_node* node, const TString& tag) { - ENSURE_NODE_NOT_EMPTY(node); - auto* str = new TString(std::move(tag)); - auto* data = new TUserDataHolder(UserData(node), str); - SetUserData(node, data); - RethrowOnError(fy_node_set_tag(node, str->c_str(), str->length()), node); -} - -bool TNodeOpsBase::RemoveTag(fy_node* node) { - ENSURE_NODE_NOT_EMPTY(node); - bool ret = fy_node_remove_tag(node); - ClearUserData(node); - return ret; -} - -bool TNodeOpsBase::HasAnchor(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return fy_node_get_anchor(node) != nullptr; -} - -void TNodeOpsBase::SetAnchor(fy_node* node, const TString& anchor) { - auto* str = new TString(anchor); - auto* data = new TUserDataHolder(UserData(node), str); - SetUserData(node, data); - RethrowOnError(fy_node_set_anchor(node, str->c_str(), str->length()), node); -} - -bool TNodeOpsBase::DeepEqual(fy_node* thisNode, fy_node* other) { - ENSURE_NODE_NOT_EMPTY(thisNode); - ENSURE_NODE_NOT_EMPTY(other); - return fy_node_compare(thisNode, other); -} - -std::unique_ptr TNodeOpsBase::EmitToCharArray(fy_node* node) const { - std::unique_ptr res( - fy_emit_node_to_string( - node, - (fy_emitter_cfg_flags)(FYECF_DEFAULT)), &FreeChar); - return res; -} - -void TNodeOpsBase::SetStyle(fy_node* node, ENodeStyle style) { - ENSURE_NODE_NOT_EMPTY(node); - fy_node_set_style(node, (enum fy_node_style)style); -} - -ENodeStyle TNodeOpsBase::Style(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return (ENodeStyle)fy_node_get_style(node); -} - -void TNodeOpsBase::SetUserData(fy_node* node, IBasicUserData* data) { - ENSURE_NODE_NOT_EMPTY(node); - fy_node_set_meta(node, data); -} - -IBasicUserData* TNodeOpsBase::UserData(fy_node* node) const { - ENSURE_NODE_NOT_EMPTY(node); - return reinterpret_cast(fy_node_get_meta(node)); -} - -void TNodeOpsBase::ClearUserData(fy_node* node) { - ENSURE_NODE_NOT_EMPTY(node); - fy_node_clear_meta(node); -} - -void ThrowAllExceptionsIfAny(fy_diag* diag) { - void* iter = nullptr; - fy_diag_error* err = fy_diag_errors_iterate(diag, &iter); - if (err != nullptr) { - TStringStream ss; - ss << err->line << ":" << err->column << " " << err->msg; - TFyamlEx ex(ss.Str()); - - while ((err = fy_diag_errors_iterate(diag, &iter)) != nullptr) { - TStringStream ss; - ss << err->line << ":" << err->column << " " << err->msg; - ex.AddError(ss.Str()); - } - - ythrow ex; - } -} - -void RethrowError(fy_diag* diag) { - void *iter = nullptr; - fy_diag_error* err; - TStringStream ss; - while ((err = fy_diag_errors_iterate(diag, &iter)) != nullptr) { - ss << err->line << ":" << err->column << " " << err->msg << "\n"; - } - ythrow TFyamlEx(ss.Str()); -} - -void RethrowOnError(bool isError, fy_node* node) { - if (!isError) { - return; - } - - std::unique_ptr diag(fy_document_get_diag(fy_node_document(node)), fy_diag_unref); - RethrowError(diag.get()); -} - -void RethrowOnError(bool isError, fy_node_pair* pair) { - if (!isError) { - return; - } - - std::unique_ptr diag(fy_document_get_diag(fy_node_document(fy_node_pair_key(pair))), fy_diag_unref); - RethrowError(diag.get()); -} - -void RethrowOnError(bool isError, fy_diag* diag) { - if (!isError) { - return; - } - - RethrowError(diag); -} - -void FreeChar(char* mem) { - free(mem); -} - -bool IsComplexType(ENodeType type) { - return type == ENodeType::Mapping || type == ENodeType::Sequence; -} - -} // namespace NDetail - -} // namespace NFyaml - -template <> -void Out(IOutputStream& out, const NFyaml::TDocument& value) { - out << value.EmitToCharArray().get(); -} - -template <> -void Out(IOutputStream& out, const NFyaml::TNodeRef& value) { - out << value.EmitToCharArray().get(); -} - -template <> -void Out(IOutputStream& out, const NFyaml::TJsonEmitter& value) { - out << value.EmitToCharArray().get(); -} - -bool operator==(const fy_node* node1, const NFyaml::NDetail::TNodeOps& node2) { - return node2.Node() == node1; -} - -bool operator==(const fy_node* node1, const NFyaml::NDetail::TNodeOps& node2) { - return node2.Node() == node1; -} diff --git a/library/cpp/yaml/fyamlcpp/fyamlcpp.h b/library/cpp/yaml/fyamlcpp/fyamlcpp.h deleted file mode 100644 index e205db71229..00000000000 --- a/library/cpp/yaml/fyamlcpp/fyamlcpp.h +++ /dev/null @@ -1,939 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include -#include - -#define ENSURE_NODE_NOT_EMPTY(NODE) Y_ENSURE_EX(NODE, TFyamlEx() << "Expected non-empty Node") -#define ENSURE_DOCUMENT_NOT_EMPTY(NODE) Y_ENSURE_EX(NODE, TFyamlEx() << "Expected non-empty Document") - -struct fy_parser; -struct fy_node; -struct fy_document; -struct fy_diag; -struct fy_document_iterator; -struct fy_node_pair; -extern "C" struct fy_node *fy_node_buildf(struct fy_document *fyd, const char *fmt, ...); - -namespace NFyaml { - namespace NDetail { - template - class TNodeOps; - } - class TNodeRef; - class TNode; -} - -bool operator==(const fy_node* node1, const NFyaml::NDetail::TNodeOps& node2); -bool operator==(const fy_node* node1, const NFyaml::NDetail::TNodeOps& node2); - -namespace NFyaml { - -struct TStringPtrHashT { - size_t operator()(const TSimpleSharedPtr& str) const { - return (size_t)str.Get(); - } -}; - -// do TFyaml(str) instead of TFyaml() << str; -class TFyamlEx : public yexception { -public: - TFyamlEx() {} - - TFyamlEx(TString error) : Errors_({error}) {} - - TFyamlEx(std::initializer_list errors) : Errors_(errors) {} - - const TVector& Errors() { - return Errors_; - } - - const char* what() const noexcept override { - What_ = TString(yexception::what()); - for (auto& err : Errors_) { - What_.push_back('\n'); - What_.append(err); - } - - return What_.c_str(); - } - - TFyamlEx& AddError(TString error) { - Errors_.push_back(error); - return *this; - } - - TStringBuf AsStrBuf() const { - return what(); - } - -private: - TVector Errors_; - mutable TString What_; -}; - -enum class ENodeType { - Scalar, - Sequence, - Mapping, -}; - -enum class ENodeStyle { - Any = -1, - Flow, - Block, - Plain, - SingleQuoted, - DoubleQuoted, - Literal, - Folded, - Alias, -}; - -class TNodeRef; -class TDocumentIterator; -class TDocument; -class TNode; -class TMappingIterator; -class TReverseMappingIterator; -class TMapping; -class TSequenceIterator; -class TReverseSequenceIterator; -class TSequence; -class TJsonEmitter; -class TParser; -struct TMark; - -namespace NDetail { - -class IBasicUserData { -public: - virtual ~IBasicUserData() = default; -}; - -template -class TUserDataHolder : public IBasicUserData { -public: - TUserDataHolder(IBasicUserData* next, T* data) - : Next_(next) - , Data_(data) - {} - -private: - std::unique_ptr Next_ = nullptr; - std::unique_ptr Data_ = nullptr; -}; - -void ThrowAllExceptionsIfAny(fy_diag* diag); - -void RethrowError(fy_diag* diag); - -void RethrowOnError(bool isError, fy_node* node); - -void RethrowOnError(bool isError, fy_node_pair* pair); - -void RethrowOnError(bool isError, fy_diag* diag); - -void FreeChar(char* mem); - -bool IsComplexType(ENodeType type); - -class TNodeOpsBase { -protected: - TString Path(fy_node* node) const; - - ENodeType Type(fy_node* node) const; - - fy_node* Copy(fy_node* node) const; - - fy_node* Copy(fy_node* node, fy_document* to) const; - - bool IsAlias(fy_node* node) const; - - fy_node* ResolveAlias(fy_node* node) const; - - fy_node* CreateReference(fy_node* node) const; - - fy_node* Sequence(fy_node* node) const; - - fy_node* Map(fy_node* node) const; - - TString Scalar(fy_node* node) const; - - TMark BeginMark(fy_node* node) const; - - TMark EndMark(fy_node* node) const; - - void Insert(fy_node* thisNode, fy_node* node); - - std::optional Tag(fy_node* node) const; - - void SetTag(fy_node* node, const TString& tag); - - bool RemoveTag(fy_node* node); - - bool HasAnchor(fy_node* node) const; - - void SetAnchor(fy_node* node, const TString& anchor); - - bool DeepEqual(fy_node* thisNode, fy_node* other); - - std::unique_ptr EmitToCharArray(fy_node* node) const; - - void SetStyle(fy_node* node, ENodeStyle style); - - ENodeStyle Style(fy_node* node) const; - -protected: - void SetUserData(fy_node* node, NDetail::IBasicUserData* data); - - NDetail::IBasicUserData* UserData(fy_node* node) const; - - void ClearUserData(fy_node* node); -}; - -template -class TNodeOps : public TNodeOpsBase { -friend class ::NFyaml::TNodeRef; - -public: - template - bool operator==(const TNodeOps& other) const { return Node() == other.Node(); } - - bool operator==(const fy_node* node) const { return Node() == node; } - - friend bool ::operator==(const fy_node* node1, const TNodeOps& node2); - friend bool ::operator==(const fy_node* node1, const TNodeOps& node2); - - explicit operator bool() const { return Node() != nullptr; } - - TString Path() const; - - ENodeType Type() const; - - TNode Copy() const; - - TNode Copy(TDocument& to) const; - - bool IsAlias() const; - - TNodeRef ResolveAlias() const; - - TNode CreateReference() const; - - TSequence Sequence() const; - - TMapping Map() const; - - TString Scalar() const; - - TMark BeginMark() const; - - TMark EndMark() const; - - void Insert(const TNodeRef& node); - - bool Empty() const { return Node() == nullptr; } - - std::optional Tag() const; - - void SetTag(const TString& tag); - - bool RemoveTag(); - - bool HasAnchor() const; - - void SetAnchor(const TString& anchor); - - bool DeepEqual(const TNodeRef& other); - - std::unique_ptr EmitToCharArray() const; - - void SetStyle(ENodeStyle style); - - ENodeStyle Style() const; - -protected: - const T& AsDerived() const; - - fy_node* Node() const; - - fy_node* Node(); - - void SetUserData(NDetail::IBasicUserData* data); - - NDetail::IBasicUserData* UserData() const; - - void ClearUserData(); -}; - -} // namespace NDetail - -class TDocumentIterator { - friend class TDocument; -public: - explicit TDocumentIterator(fy_document_iterator* iterator = nullptr); - -protected: - std::unique_ptr Iterator_; -}; - -class TNodeRef : public NDetail::TNodeOps { - friend class TNodeOps; - friend class TNodeOpsBase; - friend class TDocument; - friend class TDocumentNodeIterator; - friend class TMapping; - friend class TMappingIterator; - friend class TReverseMappingIterator; - friend class TNodePairRef; - friend class TSequence; - friend class TSequenceIterator; - friend class TReverseSequenceIterator; - friend class TJsonEmitter; - -public: - TNodeRef() = default; - - template - TNodeRef(const TNodeOps& other) : Node_(other.Node()) {} - - explicit TNodeRef(const TNodeRef& other) : Node_(other.Node_) {} - - TNodeRef(fy_node* node); - - TNodeRef& operator=(const TNodeRef& other) { Node_ = other.Node_; return *this; } - - TNodeRef& operator=(fy_node* node) { Node_ = node; return *this; } - -protected: - fy_node* Node_ = nullptr; - -private: - fy_node* NodeRawPointer() const; -}; - -class TNode : public NDetail::TNodeOps { - friend class TNodeOps; - -public: - TNode(fy_node* node = nullptr); - - template - explicit TNode(const TNodeOps& other) : Node_(other.Node_) {} - - TNodeRef Ref() { return TNodeRef(*this); } - -private: - std::shared_ptr Node_; - - TNode& operator=(fy_node* node); - - fy_node* NodeRawPointer() const { - return Node_.get(); - } -}; - -class TNodePairRef { -friend class TMappingIterator; -friend class TReverseMappingIterator; -friend class TMapping; - -public: - TNodePairRef(fy_node_pair* pair = nullptr) : Pair_(pair) {} - - bool operator==(const TNodePairRef& other) const { return Pair_ == other.Pair_; } - - explicit operator bool() const { return Pair_ != nullptr; } - - TNodeRef Key() const; - - int Index(const TNodeRef& node) const; - - void SetKey(const TNodeRef& node); - - TNodeRef Value() const; - - void SetValue(const TNodeRef& node); - -private: - fy_node_pair* Pair_ = nullptr; -}; - -class TMappingIterator { - friend class TMapping; -public: - explicit TMappingIterator(const TNodeRef& node, bool end = false); - - TMappingIterator(const TMappingIterator& other) { - Node_ = other.Node_; - NodePair_ = other.NodePair_; - } - - TMappingIterator& operator=(const TMappingIterator& other) { - Node_ = other.Node_; - NodePair_ = other.NodePair_; - return *this; - } - - TMappingIterator& operator++(); - - const TNodePairRef* operator->() const { return &NodePair_; } - - TMappingIterator operator++(int) { - TMappingIterator retval = *this; - ++(*this); - return retval; - } - - bool operator==(TMappingIterator other) const { return Node_ == other.Node_ && NodePair_ == other.NodePair_; } - - const TNodePairRef& operator*() const { return NodePair_; } - -private: - TNodeRef Node_; - TNodePairRef NodePair_; -}; - -class TReverseMappingIterator { - friend class TMapping; -public: - explicit TReverseMappingIterator(const TNodeRef& node, bool end = false); - - TReverseMappingIterator(const TReverseMappingIterator& other) { - Node_ = other.Node_; - NodePair_ = other.NodePair_; - } - - TReverseMappingIterator& operator=(const TReverseMappingIterator& other) { - Node_ = other.Node_; - NodePair_ = other.NodePair_; - return *this; - } - - TReverseMappingIterator& operator++(); - - const TNodePairRef* operator->() const { return &NodePair_; } - - TReverseMappingIterator operator++(int) { - TReverseMappingIterator retval = *this; - ++(*this); - return retval; - } - - bool operator==(TReverseMappingIterator other) const { return Node_ == other.Node_ && NodePair_ == other.NodePair_; } - - bool operator!=(TReverseMappingIterator other) const { return !(*this == other); } - - const TNodePairRef& operator*() const { return NodePair_; } - -private: - TNodeRef Node_; - TNodePairRef NodePair_; -}; - -class TMapping : public TNodeRef { -public: - template - explicit TMapping(const TNodeOps& node) - : TNodeRef(node) - { - Y_DEBUG_ABORT_UNLESS(Type() == ENodeType::Mapping); - } - - TMappingIterator begin() const { - return TMappingIterator(*this); - } - - TMappingIterator end() const { - return TMappingIterator(*this, true); - } - - TReverseMappingIterator rbegin() const { - return TReverseMappingIterator(*this); - } - - TReverseMappingIterator rend() const { - return TReverseMappingIterator(*this, true); - } - - size_t size() const; - - size_t empty() const; - - TNodePairRef at(int index) const; - - TNodePairRef operator[](int index) const; - - TNodeRef at(const TString& index) const; - - TNodePairRef pair_at(const TString& index) const; - - TNodePairRef pair_at_opt(const TString& index) const; - - TNodeRef operator[](const TString& index) const; - - TNodeRef operator[](const char* str) const; - - void Append(const TNodeRef& key, const TNodeRef& value); - - void Prepend(const TNodeRef& key, const TNodeRef& value); - - void Remove(const TNodePairRef& toRemove); - - bool Has(TString key) const; - - TMappingIterator Remove(const TMappingIterator& toRemove); - - void Remove(const TNodeRef& key); -}; - -class TSequenceIterator { - friend class TSequence; -public: - explicit TSequenceIterator(const TNodeRef& node, bool end = false); - - TSequenceIterator(const TSequenceIterator& other) { - Node_ = other.Node_; - IterNode_ = other.IterNode_; - Iter_ = other.Iter_; - } - - TSequenceIterator& operator=(const TSequenceIterator& other) { - Node_ = other.Node_; - IterNode_ = other.IterNode_; - Iter_ = other.Iter_; - return *this; - } - - TSequenceIterator& operator++(); - - const TNodeRef* operator->() const { - return &IterNode_; - } - - TSequenceIterator operator++(int) { - TSequenceIterator retval = *this; - ++(*this); - return retval; - } - - bool operator==(TSequenceIterator other) const { return Node_ == other.Node_ && Iter_ == other.Iter_; } - - bool operator!=(TSequenceIterator other) const { return !(*this == other); } - - const TNodeRef& operator*() const { return IterNode_; } - - void InsertBefore(const TNodeRef& node); - - void InsertAfter(const TNodeRef& node); - -private: - TNodeRef Node_; - TNodeRef IterNode_; - void* Iter_ = nullptr; -}; - -class TReverseSequenceIterator { - friend class TSequence; -public: - explicit TReverseSequenceIterator(const TNodeRef& node, bool end = false); - - TReverseSequenceIterator(const TReverseSequenceIterator& other) { - Node_ = other.Node_; - IterNode_ = other.IterNode_; - Iter_ = other.Iter_; - } - - TReverseSequenceIterator& operator=(const TReverseSequenceIterator& other) { - Node_ = other.Node_; - IterNode_ = other.IterNode_; - Iter_ = other.Iter_; - return *this; - } - - TReverseSequenceIterator& operator++(); - - const TNodeRef* operator->() const { - return &IterNode_; - } - - TReverseSequenceIterator operator++(int) { - TReverseSequenceIterator retval = *this; - ++(*this); - return retval; - } - - bool operator==(TReverseSequenceIterator other) const { return Node_ == other.Node_ && Iter_ == other.Iter_; } - - bool operator!=(TReverseSequenceIterator other) const { return !(*this == other); } - - const TNodeRef& operator*() const { return IterNode_; } - - void InsertBefore(const TNodeRef& node); - - void InsertAfter(const TNodeRef& node); - -private: - TNodeRef Node_; - TNodeRef IterNode_; - void* Iter_ = nullptr; -}; - -class TSequence : public TNodeRef { -public: - explicit TSequence(const TNodeRef& node) - : TNodeRef(node) - { - Y_DEBUG_ABORT_UNLESS(Type() == ENodeType::Sequence); - } - - TSequenceIterator begin() const { - return TSequenceIterator(*this); - } - - TSequenceIterator end() const { - return TSequenceIterator(*this, true); - } - - TReverseSequenceIterator rbegin() const { - return TReverseSequenceIterator(*this); - } - - TReverseSequenceIterator rend() const { - return TReverseSequenceIterator(*this, true); - } - - size_t size() const; - - size_t empty() const; - - TNodeRef at(int index) const; - - TNodeRef operator[](int index) const; - - void Append(const TNodeRef& node); - - void Prepend(const TNodeRef& node); - - void InsertBefore(const TNodeRef& mark, const TNodeRef& node); - - void InsertAfter(const TNodeRef& mark, const TNodeRef& node); - - TNode Remove(const TNodeRef& toRemove); - - TSequenceIterator Remove(const TSequenceIterator& toRemove); - - TReverseSequenceIterator Remove(const TReverseSequenceIterator& toRemove); -}; - -class TDocumentNodeIterator - : public TDocumentIterator -{ -public: - explicit TDocumentNodeIterator(TNodeRef&& node); - - TDocumentNodeIterator(const TDocumentNodeIterator& other) - : TDocumentIterator(other.Iterator_.get()) - , Node_(other.Node_) - {} - - TDocumentNodeIterator& operator=(const TDocumentNodeIterator& other) { - Iterator_.reset(other.Iterator_.get()); - Node_ = other.Node_; - return *this; - } - - TDocumentNodeIterator& operator++(); - - TNodeRef* operator->() { - return &Node_; - } - - TDocumentNodeIterator operator++(int) { - TDocumentNodeIterator retval = *this; - ++(*this); - return retval; - } - - bool operator==(TDocumentNodeIterator other) const { return Node_ == other.Node_; } - - bool operator!=(TDocumentNodeIterator other) const { return !(*this == other); } - - TNodeRef& operator*() { return Node_; } - -private: - TNodeRef Node_; -}; - -class TDocument { - friend class NDetail::TNodeOps; - friend class NDetail::TNodeOps; - friend class TParser; - - explicit TDocument(TString str, fy_document* doc = nullptr, fy_diag* diag = nullptr); - explicit TDocument(fy_document* doc = nullptr, fy_diag* diag = nullptr); - -public: - TDocument(TDocument&& other) - : Document_(std::move(other.Document_)) - , Diag_(std::move(other.Diag_)) - {} - - static TDocument Parse(TString cstr); - - TDocument Clone() const; - - template - size_t Scanf(const char* fmt, Args&& ...args) { - Y_DEBUG_ABORT_UNLESS(Document_); - return fy_document_scanf(Document_.get(), fmt, std::forward(args)...); - } - - void InsertAt(const char* path, const TNodeRef& node); - - template - TNodeRef Buildf(const char* fmt, Args&& ...args) { - Y_DEBUG_ABORT_UNLESS(Document_); - return fy_node_buildf(Document_.get(), fmt, std::forward(args)...); - } - - TNodeRef Buildf(const char* content); - - void Resolve(); - - bool HasDirectives(); - - bool HasExplicitDocumentStart(); - - bool HasExplicitDocumentEnd(); - - void SetParent(const TDocument& doc); - - TNodeRef Root(); - - void SetRoot(const TNodeRef& node); - - TDocumentNodeIterator begin() { - auto it = TDocumentNodeIterator(Root()); - ++it; - return it; - } - - TDocumentNodeIterator end() { - return TDocumentNodeIterator(TNodeRef(nullptr)); - } - - TNodeRef CreateAlias(const TString& name); - - std::unique_ptr EmitToCharArray() const; - - TMark BeginMark() const; - - TMark EndMark() const; - -private: - std::unique_ptr Document_; - std::unique_ptr Diag_; - - static void DestroyUserData(fy_node *fyn, void *meta, void *user) { - Y_UNUSED(fyn); - Y_UNUSED(user); - if (meta) { - auto* data = reinterpret_cast(meta); - delete data; - } - } - - static void DestroyDocumentStrings(fy_document *fyd, void *user) { - Y_UNUSED(fyd); - if (user) { - auto* data = reinterpret_cast, TStringPtrHashT>*>(user); - delete data; - } - } - - bool RegisterUserDataCleanup(); - void UnregisterUserDataCleanup(); -}; - -class TJsonEmitter { -public: - explicit TJsonEmitter(TDocument& doc) : Node_(doc.Root()) {} - explicit TJsonEmitter(const TNodeRef& node) : Node_(node) {} - - std::unique_ptr EmitToCharArray() const; - -private: - const TNodeRef Node_; -}; - -class TParser { - TParser(TString rawStream, fy_parser* doc, fy_diag* diag); -public: - static TParser Create(TString str); - - std::optional NextDocument(); -private: - TString RawDocumentStream_; - std::unique_ptr Parser_; - std::unique_ptr Diag_; -}; - -struct TMark { - size_t InputPos; - int Line; - int Column; -}; - -namespace NDetail { - -template -TNode TNodeOps::CreateReference() const { - return TNode(TNodeOpsBase::CreateReference(Node())); -} - -template -TNode TNodeOps::Copy() const { - return TNode(TNodeOpsBase::Copy(Node())); -} - -template -TNode TNodeOps::Copy(TDocument& to) const { - return TNode(TNodeOpsBase::Copy(Node(), to.Document_.get())); -} - -template -TString TNodeOps::Path() const { - return TNodeOpsBase::Path(Node()); -} - -template -ENodeType TNodeOps::Type() const { - return TNodeOpsBase::Type(Node()); -} - -template -bool TNodeOps::IsAlias() const { - return TNodeOpsBase::IsAlias(Node()); -} - -template -TNodeRef TNodeOps::ResolveAlias() const { - return TNodeRef(TNodeOpsBase::ResolveAlias(Node())); -} - -template -TString TNodeOps::Scalar() const { - return TNodeOpsBase::Scalar(Node()); -} - -template -TMark TNodeOps::BeginMark() const { - return TNodeOpsBase::BeginMark(Node()); -} - -template -TMark TNodeOps::EndMark() const { - return TNodeOpsBase::EndMark(Node()); -} - -template -TMapping TNodeOps::Map() const { - return TMapping(TNodeRef(TNodeOpsBase::Map(Node()))); -} - -template -TSequence TNodeOps::Sequence() const { - return TSequence(TNodeRef(TNodeOpsBase::Sequence(Node()))); -} - -template -void TNodeOps::Insert(const TNodeRef& node) { - return TNodeOpsBase::Insert(Node(), node.Node_); -} - -template -std::optional TNodeOps::Tag() const { - return TNodeOpsBase::Tag(Node()); -} - -template -void TNodeOps::SetTag(const TString& tag) { - return TNodeOpsBase::SetTag(Node(), tag); -} - -template -bool TNodeOps::RemoveTag() { - return TNodeOpsBase::RemoveTag(Node()); -} - -template -bool TNodeOps::HasAnchor() const { - return TNodeOpsBase::HasAnchor(Node()); -} - -template -void TNodeOps::SetAnchor(const TString& anchor) { - return TNodeOpsBase::SetAnchor(Node(), anchor); -} - -template -bool TNodeOps::DeepEqual(const TNodeRef& other) { - return TNodeOpsBase::DeepEqual(Node(), other.Node_); -} - -template -std::unique_ptr TNodeOps::EmitToCharArray() const { - return TNodeOpsBase::EmitToCharArray(Node()); -} - -template -void TNodeOps::SetStyle(ENodeStyle style) { - return TNodeOpsBase::SetStyle(Node(), style); -} - -template -ENodeStyle TNodeOps::Style() const { - return TNodeOpsBase::Style(Node()); -} - -template -const T& TNodeOps::AsDerived() const { - return static_cast(*this); -} - -template -fy_node* TNodeOps::Node() const { - return AsDerived().NodeRawPointer(); -} - -template -fy_node* TNodeOps::Node() { - return AsDerived().NodeRawPointer(); -} - -template -void TNodeOps::SetUserData(IBasicUserData* data) { - return TNodeOpsBase::SetUserData(Node(), data); -} - -template -IBasicUserData* TNodeOps::UserData() const { - return TNodeOpsBase::UserData(Node()); -} - -template -void TNodeOps::ClearUserData() { - return TNodeOpsBase::ClearUserData(Node()); -} - -} // namespace NDetail - -} // namesapce NFyaml diff --git a/library/cpp/yaml/fyamlcpp/ya.make b/library/cpp/yaml/fyamlcpp/ya.make deleted file mode 100644 index 06ce8066ee8..00000000000 --- a/library/cpp/yaml/fyamlcpp/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -SRCS( - fyamlcpp.cpp - fyamlcpp.h -) - -PEERDIR( - contrib/libs/libfyaml -) - -END() - -RECURSE_FOR_TESTS( - ut -)