Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into xds_tests_use_rea…
Browse files Browse the repository at this point in the history
…l_protos3
  • Loading branch information
markdroth committed Oct 30, 2024
2 parents 7b7cd34 + 55b1ae9 commit a96f342
Show file tree
Hide file tree
Showing 14 changed files with 128 additions and 42 deletions.
3 changes: 2 additions & 1 deletion bazel/grpc_build_system.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,8 @@ def grpc_cc_proto_library(name, deps = [], visibility = None):
native.cc_proto_library(name = name, deps = deps, visibility = visibility)

# DO NOT USE -- callers should instead be changed to use separate
# proto_library(), cc_proto_library(), and cc_grpc_library() rules.
# grpc_internal_proto_library(), grpc_cc_proto_library(), and
# grpc_cc_grpc_library() rules.
def grpc_proto_library(
name,
srcs = [],
Expand Down
6 changes: 3 additions & 3 deletions bazel/grpc_python_deps.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ def grpc_python_deps():
if "rules_python" not in native.existing_rules():
http_archive(
name = "rules_python",
sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
strip_prefix = "rules_python-0.26.0",
url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz",
sha256 = "be04b635c7be4604be1ef20542e9870af3c49778ce841ee2d92fcb42f9d9516a",
strip_prefix = "rules_python-0.35.0",
url = "https://github.com/bazelbuild/rules_python/releases/download/0.35.0/rules_python-0.35.0.tar.gz",
)

python_configure(name = "local_config_python")
Expand Down
9 changes: 6 additions & 3 deletions include/grpc/event_engine/event_engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,12 @@ class EventEngine : public std::enable_shared_from_this<EventEngine>,
/// on_read callback is not executed. Otherwise it returns false and the \a
/// on_read callback executes asynchronously when the read completes. The
/// caller must ensure that the callback has access to the buffer when it
/// executes. Ownership of the buffer is not transferred. Valid slices *may*
/// be placed into the buffer even if the callback is invoked with a non-OK
/// Status.
/// executes. Ownership of the buffer is not transferred. Either an error is
/// passed to the callback (like socket closed), or valid data is available
/// in the buffer, but never both at the same time. Implementations that
/// receive valid data must not throw that data away - that is, if valid
/// data is received on the underlying endpoint, a callback will be made
/// with that data available and an ok status.
///
/// There can be at most one outstanding read per Endpoint at any given
/// time. An outstanding read is one in which the \a on_read callback has
Expand Down
6 changes: 5 additions & 1 deletion include/grpc/support/metrics.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@ namespace grpc_core {
namespace experimental {

// Configuration (scope) for a specific client channel to be used for stats
// plugins.
// plugins. For some components like XdsClient where the same XdsClient instance
// can be shared across multiple channels that share the same target name but
// have different default authority and channel arguments, the component uses
// the configuration from the first channel that uses this XdsClient instance to
// determine StatsPluginChannelScope.
class StatsPluginChannelScope {
public:
StatsPluginChannelScope(
Expand Down
15 changes: 15 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,21 @@ def check_linker_need_libatomic():
return cpp_test.returncode == 0


# When building extensions for macOS on a system running macOS 10.14 or newer,
# make sure they target macOS 10.14 or newer to use C++17 stdlib properly.
# This overrides the default behavior of distutils, which targets the macOS
# version Python was built on. You can further customize the target macOS
# version by setting the MACOSX_DEPLOYMENT_TARGET environment variable before
# running setup.py.
if sys.platform == "darwin":
if "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
target_ver = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")
if target_ver == "" or tuple(int(p) for p in target_ver.split(".")) < (
10,
14,
):
os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.14"

# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
# We use these environment variables to thus get around that without locking
Expand Down
2 changes: 1 addition & 1 deletion src/core/ext/transport/chttp2/server/chttp2_server.cc
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ grpc_error_handle Chttp2ServerListener::Create(
if (args.GetBool(GRPC_ARG_ENABLE_CHANNELZ)
.value_or(GRPC_ENABLE_CHANNELZ_DEFAULT)) {
auto string_address =
grpc_event_engine::experimental::ResolvedAddressToString(addr);
grpc_event_engine::experimental::ResolvedAddressToURI(addr);
if (!string_address.ok()) {
return GRPC_ERROR_CREATE(string_address.status().ToString());
}
Expand Down
30 changes: 22 additions & 8 deletions src/core/lib/event_engine/windows/windows_endpoint.cc
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ void DumpSliceBuffer(SliceBuffer* buffer, absl::string_view context_string) {
for (size_t i = 0; i < buffer->Count(); i++) {
auto str = buffer->MutableSliceAt(i).as_string_view();
GRPC_TRACE_LOG(event_engine_endpoint, INFO)
<< context_string << ": " << str;
<< context_string << " [" << i + 1 << "/" << buffer->Count()
<< "]: " << str;
}
}

Expand Down Expand Up @@ -78,7 +79,7 @@ WindowsEndpoint::~WindowsEndpoint() {

void WindowsEndpoint::AsyncIOState::DoTcpRead(SliceBuffer* buffer) {
GRPC_TRACE_LOG(event_engine_endpoint, INFO)
<< "WindowsEndpoint::" << endpoint << " reading";
<< "WindowsEndpoint::" << endpoint << " attempting a read";
if (socket->IsShutdown()) {
socket->read_info()->SetErrorStatus(
absl::InternalError("Socket is shutting down."));
Expand Down Expand Up @@ -294,15 +295,23 @@ void WindowsEndpoint::HandleReadClosure::Run() {
return ResetAndReturnCallback()(status);
}
if (result.bytes_transferred == 0) {
DCHECK_GT(io_state.use_count(), 0);
// Either the endpoint is shut down or we've seen the end of the stream
if (GRPC_TRACE_FLAG_ENABLED(event_engine_endpoint_data)) {
DumpSliceBuffer(buffer_, absl::StrFormat("WindowsEndpoint::%p READ",
io_state->endpoint));
LOG(INFO) << "WindowsEndpoint::" << this << " read 0 bytes.";
DumpSliceBuffer(
&last_read_buffer_,
absl::StrFormat("WindowsEndpoint::%p READ last_read_buffer_: ",
io_state->endpoint));
}
status = absl::InternalError("End of TCP stream");
grpc_core::StatusSetInt(&status, grpc_core::StatusIntProperty::kRpcStatus,
GRPC_STATUS_UNAVAILABLE);
buffer_->Swap(last_read_buffer_);
if (buffer_->Length() == 0) {
// Only send an error if there is no more data to consume. If the endpoint
// or socket is shut down, the next read will discover that.
status = absl::InternalError("End of TCP stream");
grpc_core::StatusSetInt(&status, grpc_core::StatusIntProperty::kRpcStatus,
GRPC_STATUS_UNAVAILABLE);
}
return ResetAndReturnCallback()(status);
}
DCHECK_GT(result.bytes_transferred, 0);
Expand All @@ -320,8 +329,13 @@ void WindowsEndpoint::HandleReadClosure::Run() {

bool WindowsEndpoint::HandleReadClosure::MaybeFinishIfDataHasAlreadyBeenRead() {
if (last_read_buffer_.Length() > 0) {
GRPC_TRACE_LOG(event_engine_endpoint, INFO)
<< "WindowsEndpoint::" << io_state_->endpoint
<< " finishing a synchronous read";
buffer_->Swap(last_read_buffer_);
// Captures io_state_ to ensure it remains alive until the callback is run.
if (GRPC_TRACE_FLAG_ENABLED(event_engine_endpoint_data)) {
DumpSliceBuffer(buffer_, "finishing synchronous read");
}
io_state_->thread_pool->Run(
[cb = ResetAndReturnCallback()]() mutable { cb(absl::OkStatus()); });
return true;
Expand Down
40 changes: 24 additions & 16 deletions src/core/xds/grpc/xds_client_grpc.cc
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,23 @@ absl::StatusOr<std::string> GetBootstrapContents(const char* fallback_config) {
"not defined");
}

GlobalStatsPluginRegistry::StatsPluginGroup
GetStatsPluginGroupForKeyAndChannelArgs(absl::string_view key,
const ChannelArgs& channel_args) {
if (key == GrpcXdsClient::kServerKey) {
return GlobalStatsPluginRegistry::GetStatsPluginsForServer(channel_args);
}
grpc_event_engine::experimental::ChannelArgsEndpointConfig endpoint_config(
channel_args);
std::string authority =
channel_args.GetOwnedString(GRPC_ARG_DEFAULT_AUTHORITY)
.value_or(
CoreConfiguration::Get().resolver_registry().GetDefaultAuthority(
key));
experimental::StatsPluginChannelScope scope(key, authority, endpoint_config);
return GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope);
}

} // namespace

absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
Expand All @@ -241,7 +258,8 @@ absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
auto channel_args = ChannelArgs::FromC(xds_channel_args);
return MakeRefCounted<GrpcXdsClient>(
key, std::move(*bootstrap), channel_args,
MakeRefCounted<GrpcXdsTransportFactory>(channel_args));
MakeRefCounted<GrpcXdsTransportFactory>(channel_args),
GetStatsPluginGroupForKeyAndChannelArgs(key, args));
}
// Otherwise, use the global instance.
MutexLock lock(g_mu);
Expand All @@ -264,7 +282,8 @@ absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
auto channel_args = ChannelArgs::FromC(g_channel_args);
auto xds_client = MakeRefCounted<GrpcXdsClient>(
key, std::move(*bootstrap), channel_args,
MakeRefCounted<GrpcXdsTransportFactory>(channel_args));
MakeRefCounted<GrpcXdsTransportFactory>(channel_args),
GetStatsPluginGroupForKeyAndChannelArgs(key, args));
g_xds_client_map->emplace(xds_client->key(), xds_client.get());
GRPC_TRACE_LOG(xds_client, INFO) << "[xds_client " << xds_client.get()
<< "] Created xDS client for key " << key;
Expand All @@ -273,18 +292,6 @@ absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(

namespace {

GlobalStatsPluginRegistry::StatsPluginGroup GetStatsPluginGroupForKey(
absl::string_view key) {
if (key == GrpcXdsClient::kServerKey) {
return GlobalStatsPluginRegistry::GetStatsPluginsForServer(ChannelArgs{});
}
grpc_event_engine::experimental::ChannelArgsEndpointConfig endpoint_config(
ChannelArgs{});
// TODO(roth): How do we set the authority here?
experimental::StatsPluginChannelScope scope(key, "", endpoint_config);
return GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope);
}

std::string UserAgentName() {
return absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING,
GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING);
Expand All @@ -301,7 +308,8 @@ std::string UserAgentVersion() {
GrpcXdsClient::GrpcXdsClient(
absl::string_view key, std::shared_ptr<GrpcXdsBootstrap> bootstrap,
const ChannelArgs& args,
RefCountedPtr<XdsTransportFactory> transport_factory)
RefCountedPtr<XdsTransportFactory> transport_factory,
GlobalStatsPluginRegistry::StatsPluginGroup stats_plugin_group)
: XdsClient(
bootstrap, transport_factory,
grpc_event_engine::experimental::GetDefaultEventEngine(),
Expand All @@ -315,7 +323,7 @@ GrpcXdsClient::GrpcXdsClient(
certificate_provider_store_(MakeOrphanable<CertificateProviderStore>(
static_cast<const GrpcXdsBootstrap&>(this->bootstrap())
.certificate_providers())),
stats_plugin_group_(GetStatsPluginGroupForKey(key_)),
stats_plugin_group_(std::move(stats_plugin_group)),
registered_metric_callback_(stats_plugin_group_.RegisterCallback(
[this](CallbackMetricReporter& reporter) {
ReportCallbackMetrics(reporter);
Expand Down
3 changes: 2 additions & 1 deletion src/core/xds/grpc/xds_client_grpc.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ class GrpcXdsClient final : public XdsClient {
GrpcXdsClient(absl::string_view key,
std::shared_ptr<GrpcXdsBootstrap> bootstrap,
const ChannelArgs& args,
RefCountedPtr<XdsTransportFactory> transport_factory);
RefCountedPtr<XdsTransportFactory> transport_factory,
GlobalStatsPluginRegistry::StatsPluginGroup stats_plugin_group);

// Helpers for encoding the XdsClient object in channel args.
static absl::string_view ChannelArgName() {
Expand Down
15 changes: 15 additions & 0 deletions src/python/grpcio_observability/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,21 @@ def get_ext_filename(self, ext_name):
return filename


# When building extensions for macOS on a system running macOS 10.14 or newer,
# make sure they target macOS 10.14 or newer to use C++17 stdlib properly.
# This overrides the default behavior of distutils, which targets the macOS
# version Python was built on. You can further customize the target macOS
# version by setting the MACOSX_DEPLOYMENT_TARGET environment variable before
# running setup.py.
if sys.platform == "darwin":
if "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
target_ver = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")
if target_ver == "" or tuple(int(p) for p in target_ver.split(".")) < (
10,
14,
):
os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.14"

# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
# We use these environment variables to thus get around that without locking
Expand Down
3 changes: 0 additions & 3 deletions test/core/end2end/end2end_tests.cc
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@ void CoreEnd2endTest::SetUp() {
CoreConfiguration::Reset();
initialized_ = false;
grpc_prewarm_os_for_tests();
#ifdef GPR_WINDOWS
GTEST_SKIP() << "Disabled on Windows due to high flake rate";
#endif
}

void CoreEnd2endTest::TearDown() {
Expand Down
21 changes: 17 additions & 4 deletions test/cpp/end2end/xds/xds_core_end2end_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1115,10 +1115,23 @@ TEST_P(XdsFederationTest, FederationServer) {
class XdsMetricsTest : public XdsEnd2endTest {
protected:
void SetUp() override {
stats_plugin_ = grpc_core::FakeStatsPluginBuilder()
.UseDisabledByDefaultMetrics(true)
.BuildAndRegister();
InitClient();
stats_plugin_ =
grpc_core::FakeStatsPluginBuilder()
.UseDisabledByDefaultMetrics(true)
.SetChannelFilter(
[](const grpc_core::experimental::StatsPluginChannelScope&
scope) {
return scope.target() == absl::StrCat("xds:", kServerName) &&
scope.default_authority() == kServerName &&
scope.experimental_args().GetString("test_only.arg") ==
"test_only.value";
})
.BuildAndRegister();
ChannelArguments args;
args.SetString("test_only.arg", "test_only.value");
InitClient(/*builder=*/absl::nullopt, /*lb_expected_authority=*/"",
/*xds_resource_does_not_exist_timeout_ms=*/0,
/*balancer_authority_override=*/"", /*args=*/&args);
}

std::shared_ptr<grpc_core::FakeStatsPlugin> stats_plugin_;
Expand Down
2 changes: 1 addition & 1 deletion test/cpp/end2end/xds/xds_end2end_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,7 @@ TEST_P(XdsSecurityTest, MtlsWithAggregateCluster) {
EXPECT_EQ(backends_[0]->backend_service()->last_peer_identity(),
authenticated_identity_);
// Now stop backend 0 and wait for backend 1.
ShutdownBackend(0);
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
// Make sure the backend saw the right client identity.
EXPECT_EQ(backends_[1]->backend_service()->last_peer_identity(),
Expand Down
15 changes: 15 additions & 0 deletions tools/distrib/python/grpcio_tools/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,21 @@ def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
build_ext.build_ext.build_extensions(self)


# When building extensions for macOS on a system running macOS 10.14 or newer,
# make sure they target macOS 10.14 or newer to use C++17 stdlib properly.
# This overrides the default behavior of distutils, which targets the macOS
# version Python was built on. You can further customize the target macOS
# version by setting the MACOSX_DEPLOYMENT_TARGET environment variable before
# running setup.py.
if sys.platform == "darwin":
if "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
target_ver = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")
if target_ver == "" or tuple(int(p) for p in target_ver.split(".")) < (
10,
14,
):
os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.14"

# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
# We use these environment variables to thus get around that without locking
Expand Down

0 comments on commit a96f342

Please sign in to comment.