constHistogram write cause "concurrent map iteration and map write" panic

[sunlintong]

version: v1.11.0

fatal error: concurrent map iteration and map write{"level":"debug","ts":1639148806.1929348,"msg":"collector succeeded, name: request, duration_seconds 0.000006"}


goroutine 106 [running]:
runtime.throw({0x29f51a4, 0x7f5e2c9cfd18})
	/usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc000bdace8 sp=0xc000bdacb8 pc=0x4346f1
runtime.mapiternext(0xc000bdade8)
	/usr/local/go/src/runtime/map.go:858 +0x4eb fp=0xc000bdad58 sp=0xc000bdace8 pc=0x40f70b
github.com/prometheus/client_golang/prometheus.(*constHistogram).Write(0xc0040df340, 0xc00339f180)
	/builds/6QV6Lu47/0/gemscloud/service/.go/pkg/mod/github.com/prometheus/[email protected]/prometheus/histogram.go:561 +0x150 fp=0xc000bdae58 sp=0xc000bdad58 pc=0x1ee3df0
github.com/prometheus/client_golang/prometheus.processMetric({0x2ec7660, 0xc0040df340}, 0x0, 0x0, 0x0)
	/builds/6QV6Lu47/0/gemscloud/service/.go/pkg/mod/github.com/prometheus/[email protected]/prometheus/registry.go:598 +0x98 fp=0xc000bdaf80 sp=0xc000bdae58 pc=0x1ee87d8
github.com/prometheus/client_golang/prometheus.(*Registry).Gather(0xc0000ff950)
	/builds/6QV6Lu47/0/gemscloud/service/.go/pkg/mod/github.com/prometheus/[email protected]/prometheus/registry.go:492 +0x7ba fp=0xc000bdb450 sp=0xc000bdaf80 pc=0x1ee7efa
github.com/prometheus/client_golang/prometheus.Gatherers.Gather({0xc000fe2460, 0x2, 0x0})
	/builds/6QV6Lu47/0/gemscloud/service/.go/pkg/mod/github.com/prometheus/[email protected]/prometheus/registry.go:719 +0x217 fp=0xc000bdb7a0 sp=0xc000bdb450 pc=0x1ee9497
github.com/prometheus/client_golang/prometheus.(*Gatherers).Gather(0xc00014dd10)
	<autogenerated>:1 +0x3a fp=0xc000bdb7c8 sp=0xc000bdb7a0 pc=0x1ef157a
github.com/prometheus/client_golang/prometheus/promhttp.HandlerFor.func1({0x2ee46d8, 0xc004256e00}, 0xc002cd4700)
	/builds/6QV6Lu47/0/gemscloud/service/.go/pkg/mod/github.com/prometheus/[email protected]/prometheus/promhttp/http.go:126 +0x108 fp=0xc000bdb968 sp=0xc000bdb7c8 pc=0x1ef32e8
net/http.HandlerFunc.ServeHTTP(0x0, {0x2ee46d8, 0xc004256e00}, 0xc00048c288)
	/usr/local/go/src/net/http/server.go:2046 +0x2f fp=0xc000bdb990 sp=0xc000bdb968 pc=0x6eabcf
cloudminds.com/gemscloud/service/pkg/utils/exporter.(*handler).ServeHTTP(0xc000cf8d20, {0x2ee46d8, 0xc004256e00}, 0xc002cd4700)
	/builds/6QV6Lu47/0/gemscloud/service/pkg/utils/exporter/exporter.go:82 +0x2d5 fp=0xc000bdba70 sp=0xc000bdb990 pc=0x1f1d555
net/http.(*ServeMux).ServeHTTP(0x0, {0x2ee46d8, 0xc004256e00}, 0xc002cd4700)
	/usr/local/go/src/net/http/server.go:2424 +0x149 fp=0xc000bdbac0 sp=0xc000bdba70 pc=0x6ec989
net/http.serverHandler.ServeHTTP({0xc001081260}, {0x2ee46d8, 0xc004256e00}, 0xc002cd4700)
	/usr/local/go/src/net/http/server.go:2878 +0x43b fp=0xc000bdbb80 sp=0xc000bdbac0 pc=0x6ee5fb
net/http.(*conn).serve(0xc000797c20, {0x2efc4a8, 0xc000fb7c20})
	/usr/local/go/src/net/http/server.go:1929 +0xb08 fp=0xc000bdbfb8 sp=0xc000bdbb80 pc=0x6e9ca8
net/http.(*Server).Serve·dwrap·87()
	/usr/local/go/src/net/http/server.go:3033 +0x2e fp=0xc000bdbfe0 sp=0xc000bdbfb8 pc=0x6eef4e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc000bdbfe8 sp=0xc000bdbfe0 pc=0x4655c1
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3033 +0x4e8

The code may be here:

client_golang/prometheus/histogram.go

Lines 554 to 577 in 8184d76

	func (h *constHistogram) Write(out *dto.Metric) error {
	his := &dto.Histogram{}
	buckets := make([]*dto.Bucket, 0, len(h.buckets))
	his.SampleCount = proto.Uint64(h.count)
	his.SampleSum = proto.Float64(h.sum)
	for upperBound, count := range h.buckets {
	buckets = append(buckets, &dto.Bucket{
	CumulativeCount: proto.Uint64(count),
	UpperBound: proto.Float64(upperBound),
	})
	}
	if len(buckets) > 0 {
	sort.Sort(buckSort(buckets))
	}
	his.Bucket = buckets
	out.Histogram = his
	out.Label = h.labelPairs
	return nil
	}

[sunlintong]

I think this may happens when "func (h *constHistogram) Write(out *dto.Metric) error" and my function which wirte data in buckets at the same time.
I can copy a new buckets to resolve this problem, is there other solutions?

[beorn7]

Indeed, the observed behavior is expected. NewConstHistogram creates a "throw-away" histogram for one-time use (that's what the "constant" in the name tries to allude to). Changing the buckets map after you have handed over the histogram in a Collect call is a race.