From c49dda68d20c06ffadfee0239e0737ba958826ee Mon Sep 17 00:00:00 2001 From: Nayef Ghattas Date: Tue, 9 Jul 2024 15:10:23 +0000 Subject: [PATCH] symbolication: separate extraction from upload timeouts we're seeing sporadic timeouts on internal fleets, splitting timeouts to give both the extraction and the upload a bit more headroom --- symbolication/datadog_uploader.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/symbolication/datadog_uploader.go b/symbolication/datadog_uploader.go index b26c0fe..d57c4b9 100644 --- a/symbolication/datadog_uploader.go +++ b/symbolication/datadog_uploader.go @@ -29,6 +29,7 @@ const binaryCacheSize = 1000 const sourceMapEndpoint = "/api/v2/srcmap" +const symbolExtractionTimeout = 10 * time.Second const uploadTimeout = 10 * time.Second type DatadogUploader struct { @@ -117,15 +118,12 @@ func (d *DatadogUploader) HandleExecutable(elfRef *pfelf.Reference, fileID libpf // if there are many executables. // Ideally, we should limit the number of concurrent uploads go func() { - ctx, cancel := context.WithTimeout(context.Background(), uploadTimeout) - defer cancel() - if d.dryRun { log.Infof("Dry run: would upload symbols %s for executable: %s", inputFilePath, e) return } - err = d.handleSymbols(ctx, inputFilePath, e) + err = d.handleSymbols(inputFilePath, e) if err != nil { d.uploadCache.Remove(fileID) log.Errorf("Failed to handle symbols: %v for executable: %s", err, e) @@ -184,7 +182,7 @@ func (e *executableMetadata) String() string { ) } -func (d *DatadogUploader) handleSymbols(ctx context.Context, symbolPath string, +func (d *DatadogUploader) handleSymbols(symbolPath string, e *executableMetadata) error { symbolFile, err := os.CreateTemp("", "objcopy-debug") if err != nil { @@ -193,11 +191,15 @@ func (d *DatadogUploader) handleSymbols(ctx context.Context, symbolPath string, defer os.Remove(symbolFile.Name()) defer symbolFile.Close() + ctx, cancel := context.WithTimeout(context.Background(), symbolExtractionTimeout) + defer cancel() err = d.copySymbols(ctx, symbolPath, symbolFile.Name()) if err != nil { return fmt.Errorf("failed to copy symbols: %w", err) } + ctx, cancel = context.WithTimeout(context.Background(), uploadTimeout) + defer cancel() err = d.uploadSymbols(ctx, symbolFile, e) if err != nil { return fmt.Errorf("failed to upload symbols: %w", err)