Skip to content

Commit

Permalink
use sample rate for all seer metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@lobsterkatie
lobsterkatie committed Aug 3, 2024
1 parent c1f7f43 commit 808fd0f
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 31 deletions.
14 changes: 6 additions & 8 deletions src/sentry/grouping/ingest/seer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def _has_customized_fingerprint(event: Event, primary_hashes: CalculatedHashes)
else:
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": "hybrid-fingerprint"},
)
return True
Expand All @@ -102,7 +102,7 @@ def _has_customized_fingerprint(event: Event, primary_hashes: CalculatedHashes)
if fingerprint_variant:
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": fingerprint_variant.type},
)
return True
Expand Down Expand Up @@ -133,7 +133,7 @@ def _ratelimiting_enabled(event: Event, project: Project) -> bool:
)
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": "global-rate-limit"},
)

Expand All @@ -151,7 +151,7 @@ def _ratelimiting_enabled(event: Event, project: Project) -> bool:
)
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": "project-rate-limit"},
)

Expand Down Expand Up @@ -179,7 +179,7 @@ def _circuit_breaker_broken(event: Event, project: Project) -> bool:
)
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": "circuit-breaker"},
)

Expand Down Expand Up @@ -251,9 +251,7 @@ def maybe_check_seer_for_matching_grouphash(
if should_call_seer_for_grouping(event, primary_hashes):
metrics.incr(
"grouping.similarity.did_call_seer",
# TODO: Consider lowering this (in all the spots this metric is
# collected) once we roll Seer grouping out more widely
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": True, "blocker": "none"},
)
try:
Expand Down
25 changes: 21 additions & 4 deletions src/sentry/seer/similarity/grouping_records.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from django.conf import settings
from urllib3.exceptions import ReadTimeoutError

from sentry import options
from sentry.conf.server import (
SEER_GROUPING_RECORDS_URL,
SEER_HASH_GROUPING_RECORDS_DELETE_URL,
Expand Down Expand Up @@ -106,13 +107,21 @@ def delete_project_grouping_records(
"seer.delete_grouping_records.project.success",
extra={"project_id": project_id},
)
metrics.incr("grouping.similarity.delete_records_by_project", tags={"success": True})
metrics.incr(
"grouping.similarity.delete_records_by_project",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"success": True},
)
return True
else:
logger.error(
"seer.delete_grouping_records.project.failure",
)
metrics.incr("grouping.similarity.delete_records_by_project", tags={"success": False})
metrics.incr(
"grouping.similarity.delete_records_by_project",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"success": False},
)
return False


Expand Down Expand Up @@ -140,9 +149,17 @@ def delete_grouping_records_by_hash(project_id: int, hashes: list[str]) -> bool:
"seer.delete_grouping_records.hashes.success",
extra=extra,
)
metrics.incr("grouping.similarity.delete_records_by_hash", tags={"success": True})
metrics.incr(
"grouping.similarity.delete_records_by_hash",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"success": True},
)
return True
else:
logger.error("seer.delete_grouping_records.hashes.failure", extra=extra)
metrics.incr("grouping.similarity.delete_records_by_hash", tags={"success": False})
metrics.incr(
"grouping.similarity.delete_records_by_hash",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"success": False},
)
return False
17 changes: 6 additions & 11 deletions src/sentry/seer/similarity/similar_issues.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,6 @@

logger = logging.getLogger(__name__)

# TODO: Keeping this at a 100% sample rate for now in order to get good signal as we're rolling out
# and calls are still comparatively rare. Once traffic gets heavy enough, we should probably ramp
# this down.
SIMILARITY_REQUEST_METRIC_SAMPLE_RATE = 1.0


seer_grouping_connection_pool = connection_from_url(
settings.SEER_GROUPING_URL,
Expand Down Expand Up @@ -93,7 +88,7 @@ def get_similarity_data_from_seer(
)
metrics.incr(
"seer.similar_issues_request",
sample_rate=SIMILARITY_REQUEST_METRIC_SAMPLE_RATE,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={
**metric_tags,
"outcome": "empty_stacktrace",
Expand All @@ -119,7 +114,7 @@ def get_similarity_data_from_seer(
logger.warning("get_seer_similar_issues.request_error", extra=logger_extra)
metrics.incr(
"seer.similar_issues_request",
sample_rate=SIMILARITY_REQUEST_METRIC_SAMPLE_RATE,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={**metric_tags, "outcome": "error", "error": type(e).__name__},
)
circuit_breaker.record_error()
Expand All @@ -141,7 +136,7 @@ def get_similarity_data_from_seer(

metrics.incr(
"seer.similar_issues_request",
sample_rate=SIMILARITY_REQUEST_METRIC_SAMPLE_RATE,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={
**metric_tags,
"outcome": "error",
Expand Down Expand Up @@ -171,15 +166,15 @@ def get_similarity_data_from_seer(
)
metrics.incr(
"seer.similar_issues_request",
sample_rate=SIMILARITY_REQUEST_METRIC_SAMPLE_RATE,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={**metric_tags, "outcome": "error", "error": type(e).__name__},
)
return []

if not response_data:
metrics.incr(
"seer.similar_issues_request",
sample_rate=SIMILARITY_REQUEST_METRIC_SAMPLE_RATE,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={**metric_tags, "outcome": "no_similar_groups"},
)
return []
Expand Down Expand Up @@ -241,7 +236,7 @@ def get_similarity_data_from_seer(

metrics.incr(
"seer.similar_issues_request",
sample_rate=SIMILARITY_REQUEST_METRIC_SAMPLE_RATE,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags=metric_tags,
)
return sorted(
Expand Down
6 changes: 3 additions & 3 deletions src/sentry/seer/similarity/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def get_stacktrace_string(data: dict[str, Any]) -> str:

metrics.incr(
"seer.grouping.html_in_stacktrace",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={
"html_frames": (
"none"
Expand Down Expand Up @@ -161,7 +161,7 @@ def killswitch_enabled(project_id: int, event: Event | None = None) -> bool:
metrics.incr("grouping.similarity.seer_global_killswitch_enabled")
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": "global-killswitch"},
)
return True
Expand All @@ -174,7 +174,7 @@ def killswitch_enabled(project_id: int, event: Event | None = None) -> bool:
metrics.incr("grouping.similarity.seer_similarity_killswitch_enabled")
metrics.incr(
"grouping.similarity.did_call_seer",
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
tags={"call_made": False, "blocker": "similarity-killswitch"},
)
return True
Expand Down
22 changes: 17 additions & 5 deletions src/sentry/tasks/embeddings_grouping/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,10 @@ def get_data_from_snuba(project, groups_to_backfill_with_no_embedding):
)
snuba_requests.append(request)

with metrics.timer(f"{BACKFILL_NAME}.bulk_snuba_queries", sample_rate=1.0):
with metrics.timer(
f"{BACKFILL_NAME}.bulk_snuba_queries",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
):
snuba_results_chunk = _make_snuba_call(
project, snuba_requests, Referrer.GROUPING_RECORDS_BACKFILL_REFERRER.value
)
Expand Down Expand Up @@ -368,7 +371,10 @@ def _make_seer_call(


@sentry_sdk.tracing.trace
@metrics.wraps(f"{BACKFILL_NAME}.send_group_and_stacktrace_to_seer", sample_rate=1.0)
@metrics.wraps(
f"{BACKFILL_NAME}.send_group_and_stacktrace_to_seer",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
)
def send_group_and_stacktrace_to_seer(
groups_to_backfill_with_no_embedding_has_snuba_row_and_nodestore_row,
nodestore_results,
Expand All @@ -386,7 +392,10 @@ def send_group_and_stacktrace_to_seer(


@sentry_sdk.tracing.trace
@metrics.wraps(f"{BACKFILL_NAME}.send_group_and_stacktrace_to_seer", sample_rate=1.0)
@metrics.wraps(
f"{BACKFILL_NAME}.send_group_and_stacktrace_to_seer",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
)
def send_group_and_stacktrace_to_seer_multithreaded(
groups_to_backfill_with_no_embedding_has_snuba_row_and_nodestore_row,
nodestore_results,
Expand Down Expand Up @@ -544,7 +553,10 @@ def process_chunk(chunk):
return bulk_data


@metrics.wraps(f"{BACKFILL_NAME}.lookup_event_bulk", sample_rate=1.0)
@metrics.wraps(
f"{BACKFILL_NAME}.lookup_event_bulk",
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
)
@sentry_sdk.tracing.trace
def lookup_group_data_stacktrace_bulk(
project: Project, rows: list[GroupEventRow]
Expand Down Expand Up @@ -608,7 +620,7 @@ def lookup_group_data_stacktrace_bulk(
metrics.gauge(
f"{BACKFILL_NAME}._lookup_event_bulk.hit_ratio",
round(len(bulk_data.items()) / len(rows)) * 100,
sample_rate=1.0,
sample_rate=options.get("seer.similarity.metrics_sample_rate"),
)

return groups_to_event
Expand Down

0 comments on commit 808fd0f

Please sign in to comment.