From 0de523469580cbe67fac366e2728298ce20b41bf Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 12:26:32 +0200 Subject: [PATCH 1/8] Improve multi-arch build + documentation --- public-docker-images/Dockerfile-alpine | 12 ++++++------ public-docker-images/Dockerfile-debian-sid | 9 +++++---- public-docker-images/README.md | 14 +++++++++++--- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/public-docker-images/Dockerfile-alpine b/public-docker-images/Dockerfile-alpine index 04814cec5..384f185f2 100644 --- a/public-docker-images/Dockerfile-alpine +++ b/public-docker-images/Dockerfile-alpine @@ -1,11 +1,11 @@ -FROM golang:1.18-alpine3.16 AS build +FROM --platform=$BUILDPLATFORM golang:1.18-alpine3.16 AS build WORKDIR /veneur/ COPY . . -RUN CGO_ENABLED=0 go build -a -ldflags "-X github.com/stripe/veneur/v14/util/build.VERSION=${VERSION} -X github.com/stripe/veneur/v14/util/build.BUILD_DATE=$(date +%s)" -o /build/veneur ./cmd/veneur &&\ - CGO_ENABLED=0 go build -a -o /build/veneur-emit ./cmd/veneur-emit &&\ - CGO_ENABLED=0 go build -a -o /build/veneur-prometheus ./cmd/veneur-prometheus &&\ - CGO_ENABLED=0 go build -a -ldflags "-X github.com/stripe/veneur/v14/util/build.VERSION=${VERSION} -X github.com/stripe/veneur/v14/util/build.BUILD_DATE=$(date +%s)" -o /build/veneur-proxy ./cmd/veneur-proxy - +ARG TARGETOS TARGETARCH +RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -ldflags "-X github.com/stripe/veneur/v14/util/build.VERSION=${VERSION} -X github.com/stripe/veneur/v14/util/build.BUILD_DATE=$(date +%s)" -o /build/veneur ./cmd/veneur &&\ + GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -o /build/veneur-emit ./cmd/veneur-emit &&\ + GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -o /build/veneur-prometheus ./cmd/veneur-prometheus &&\ + GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -ldflags "-X github.com/stripe/veneur/v14/util/build.VERSION=${VERSION} -X github.com/stripe/veneur/v14/util/build.BUILD_DATE=$(date +%s)" -o /build/veneur-proxy ./cmd/veneur-proxy FROM alpine:3.16 AS release LABEL maintainer="The Stripe Observability Team " diff --git a/public-docker-images/Dockerfile-debian-sid b/public-docker-images/Dockerfile-debian-sid index 5f8c3239a..22c27ea92 100644 --- a/public-docker-images/Dockerfile-debian-sid +++ b/public-docker-images/Dockerfile-debian-sid @@ -1,10 +1,11 @@ FROM golang:1.18 AS build WORKDIR /veneur/ COPY . . -RUN go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur ./cmd/veneur &&\ - go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur-emit ./cmd/veneur-emit &&\ - go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur-prometheus ./cmd/veneur-prometheus &&\ - go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur-proxy ./cmd/veneur-proxy +ARG TARGETOS TARGETARCH +RUN GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur ./cmd/veneur &&\ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur-emit ./cmd/veneur-emit &&\ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur-prometheus ./cmd/veneur-prometheus &&\ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -ldflags "-X github.com/stripe/veneur.VERSION=${VERSION}" -o /build/veneur-proxy ./cmd/veneur-proxy FROM debian:sid AS release diff --git a/public-docker-images/README.md b/public-docker-images/README.md index 5ebd673f4..41dc4fcd5 100644 --- a/public-docker-images/README.md +++ b/public-docker-images/README.md @@ -3,16 +3,23 @@ This folder holds the Docker resources for building [Veneur](https://github.com/stripe/veneur), and a sample systemd unit for running it. ## Building + +You should run these commands from the project root. + For the Debian-based image: + ``` -docker build --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid . +docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur-local -f public-docker-images/Dockerfile-debian-sid --pull --push . ``` For the Alpine Linux-based image: + ``` -docker build --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine . +docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur-local -f public-docker-images/Dockerfile-alpine --pull --push . ``` +For both cases you could remove ```--platform` arugment if you just plan build for the host architechture. + ## Running The example.yaml comes built-in with the docker image, but won't be usefully functional without supplying a few [configuration options](https://github.com/stripe/veneur#configuration-via-environment-variables) and run arguments. @@ -40,6 +47,7 @@ $ docker run --rm -it \ ## Veneur-emit `veneur-emit` is also included in the image, and can be run explicitly like so: + ``` -$ docker run --net=host -it veneur /veneur/veneur-emit -count 1 -name foo -hostport "127.0.0.1:8126" +docker run --net=host -it veneur /veneur/veneur-emit -count 1 -name foo -hostport "127.0.0.1:8126" ``` From 7a2b4a1693d96030525dc9fde70617a782ae22ec Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 12:37:59 +0200 Subject: [PATCH 2/8] Add entry to changelog --- CHANGELOG.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 100a2d75d..ccbc115ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # 14.2.0, UNRELEASED ## Added + * A flag -print-secrets to disable redacting config secrets. * A prometheus remote-write sink compatible with Cortex and more. Thanks, [philipnrmn](https://github.com/philipnrmn)! * Some veneur-prometheus arguments to rename and add additional tags. Thanks, [christopherb-stripe](https://github.com/christopherb-stripe)! @@ -10,10 +11,13 @@ * `trace.StartTrace` and `trace.StartChildSpan` now scale better across multiple goroutines. Thanks [bpowers](https://github.com/bpowers) ## Updated + * Use `T.TempDir` to create temporary directory in tests ([#944](https://github.com/stripe/veneur/pull/944)). -* When the request to send data from Cloudwatch & SFX sink fails, log the count of metrics that are dropped. +* When the request to send data from Cloudwatch & SFX sink fails, log the count of metrics that are dropped. +* Improve multi-arch build by making use golang cross-compilation ([#999](https://github.com/stripe/veneur/pull/990)) ## Bugfixes + * A fix for forwarding metrics with gRPC using the kubernetes discoverer. Thanks, [androohan](https://github.com/androohan)! * Regenerate testing certs/CA that have expired and have broken tests. Thanks, [randallm](https://github.com/randallm) * The config field `trace_lightstep_access_token` is redacted if printed. Thanks [arnavdugar](https://github.com/arnavdugar)! @@ -37,6 +41,7 @@ # 14.0.0, 2020-01-14 ## Updated + * Migrated from dep to Go modules. Clients must now use the updated import path `github.com/stripe/veneur/v14`. Thanks, [andybons](https://github.com/andybons)! # 13.0.0, 2020-01-05 @@ -69,17 +74,19 @@ * Updated the vendored version of DataDog/datadog-go which adds support for sending metrics to Unix Domain socket. Thanks, [prudhvi](https://github.com/prudhvi)! * Splunk sink: Downgraded Splunk HEC errors to be logged at warning level, rather than error level. Added a note to clarify that Splunk cluster restarts can cause temporary errors, which are not necessarily problematic. Thanks, [aditya](https://github.com/chimeracoder)! * Updated the vendored version of github.com/gogo/protobuf which fixes Gopkg.toml conflicts for users of veneur. Thanks, [dtbartle](http://github.com/dtbartle)! -* Updated server.go to use the aws sdk (https://docs.aws.amazon.com/sdk-for-go/api/aws/session/) when the creds are not set in the config.yaml. Thanks, [linuxdynasty](https://github.com/linuxdynasty)! +* Updated server.go to use the aws sdk () when the creds are not set in the config.yaml. Thanks, [linuxdynasty](https://github.com/linuxdynasty)! * Changed the certificates that veneur tests with to include SANs and no longer rely on Common Names, in order to comply with Go's [upcoming crackdown on CN certificate constraints](https://github.com/stripe/veneur/issues/791). Thanks, [antifuchs](https://github.com/antifuchs)! * Disabled the dogstatsd client telemetry on the internal statsd client used by Veneur. Thanks, [prudhvi](https://github.com/prudhvi)! * Migrated from the deprecated Sentry package, raven-go, to sentry-go. Thanks, [yanske](https://github.com/yanske)! ## Bugfixes + * veneur-prometheus now reports incremental counters instead of cumulative counters. This may cause dramatic differences in the statistics reported by veneur-prometheus. Thanks, [kklipsch-stripe](https://github.com/kklipsch-stripe)! * veneur-emit no longer panics when an empty command is passed. Thanks, [shrivu-stripe](https://github.com/shrivu-stripe)! * Fixed a bug that caused some veneur-emit builds to not flush metrics to udp. Thanks, [shrivu-stripe](https://github.com/shrivu-stripe)! ## Bugfixes + * Veneur listening on UDS for statsd metrics will respect the `read_buffer_size_bytes` config. Thanks, [prudhvi](https://github.com/prudhvi)! * The splunk HEC span sink didn't correctly spawn the number of submission workers configured with `splunk_hec_submission_workers`, only spawning one. Now it spawns the number configured. Thanks, [antifuchs](https://github.com/antifuchs)! * The signalfx sink now correctly constructs ingestion endpoint URLs when given URLs that end in slashes. Thanks, [antifuchs](https://github.com/antifuchs)! @@ -102,21 +109,25 @@ * Support for listening to statsd metrics on Unix Domain Socket(Datagram type). Thanks, [prudhvi](https://github.com/prudhvi)! ## Updated + * The metric `veneur.sink.spans_dropped_total` now includes packets that were skipped due to UDP write errors. Thanks, [aditya](https://github.com/chimeracoder)! * The `debug` blackhole sink features improved logging output, with more data and better formatting. Thanks, [aditya](https://github.com/chimeracoder)! * Container images are now built with Go 1.12. Thanks, [aditya](https://github.com/chimeracoder)! ## Bugfixes + * The signalfx client no longer reports a timeout when submission to the datapoint API endpoint encounters an error. Thanks, [antifuchs](https://github.com/antifuchs)! * SSF packets without a name are no longer considered valid for `protocol.ValidTrace`. Thanks, [tummychow](https://github.com/tummychow)! * The splunk sink no longer hangs or complains when a HEC endpoint should close the connection. Thanks, [antifuchs](https://github.com/antifuchs)! ## Removed + * Go 1.10 is no longer supported. # 11.0.0, 2019-01-22 ## Added + * Datadog's [distribution](https://docs.datadoghq.com/developers/metrics/distributions/) type for DogStatsD is now supported and treated as a plain histogram for compatibility. Thanks, [gphat](https://github.com/gphat)! * Add support for `tags_exclude` to the DataDog metrics sink. Thanks, [mhamrah](https://github.com/mhamrah)! * The `github.com/stripe/veneur/trace` package has brand new and much more extensive [documentation](https://godoc.org/github.com/stripe/veneur/trace)! Thanks, [antifuchs](https://github.com/antifuchs)! @@ -125,21 +136,25 @@ # 10.0.0, 2018-12-19 ## Added + * The new X-Ray sink provides support for [AWS X-Ray](https://aws.amazon.com/xray/) as a tracing backend. Thanks, [gphat](https://github.com/gphat) and [aditya](https://github.com/chimeracoder)! * A new package `github.com/stripe/veneur/trace/testbackend` contains two trace client backends that can be used to test the trace data emitted by applications. Thanks, [antifuchs](https://github.com/antifuchs)! ## Updated + * Updated the vendored version of x/net, which picks up a package rename that can lead issues when integrating veneur into other codebases. Thanks, [nicktrav](https://github.com/nicktrav)! * Updated the vendored versions of x/sys, protobuf, and gRPC. Thanks [nicktrav](https://github.com/nicktrav)! # 9.0.0, 2018-11-08 ## Bugfixes + * The Splunk span sink no longer reports an internal error for timeouts encountered in event submissions; instead, it reports a failure metric with a cause tag set to `submission_timeout`. Thanks, [antifuchs](https://github.com/antifuchs)! * The Splunk span sink now honors `Connection: keep-alive` from the HEC endpoint and keeps around as many idle HTTP connections in reserve as it has HEC submission workers. Thanks, [antifuchs](https://github.com/antifuchs)! * The metric `veneur.forward.post_metrics_total` was being emitted both as a gauge and a counter. The errant gauge was removed. Thanks, [gphat](https://github.com/gphat)! ## Added + * The Splunk span sink can be configured with a sample rate for non-indicator spans with the `splunk_span_sample_rate` setting. Thanks, [aditya](https://github.com/chimeracoder)! * The splunk span sink now has configuration parameters `splunk_hec_max_connection_lifetime` and `splunk_hec_connection_lifetime_jitter` to regulate how long HTTP connections can be kept alive for. Thanks, [antifuchs](https://github.com/antifuchs)! * The SignalFx sink can now filter metric names by prefix with `signalfx_metric_name_prefix_drops` and tag literals (case-insensitive) with `signalfx_metric_tag_literal_drops`. Thanks [gphat](https://github.com/gphat)! @@ -147,10 +162,12 @@ * The `ssf.spans.root.received_total` global counter tracks the number of traces (root spans) processed system-wide. Thanks, [aditya](https://github.com/chimeracoder)! ## Updated + * The README's [Metrics section](https://github.com/stripe/veneur#metrics) has been updated, as it referred to some missing metrics. Thanks, [gphat](https://github.com/gphat)! * Various references to Datadog were removed from the README, Veneur is vendor agnostic. Thanks, [gphat](https://github.com/gphat)! ## Removed + * The metrics `veneur.flush.total_duration_ns` and `veneur.flush.worker_duration_ns` were removed, please use the per-sink `veneur.sink.metric_flush_total_duration_ns` to monitor flush durations. * The metrics `veneur.gc.GCCPUFraction`, `veneur.gc.alloc_heap_bytes_total`, `veneur.gc.mallocs_objects_total` metrics were removed. Also from veneur proxy. Thanks, [gphat](https://github.com/gphat)! * The metric `veneur.flush.other_samples_duration_ns` was removed. Thanks, [gphat](https://github.com/gphat)! @@ -158,6 +175,7 @@ # 8.0.0, 2018-09-20 ## Added + * Metrics can be forwarded over gRPC using veneur-proxy (and Consul). Thanks, [noahgoldman](https://github.com/noahgoldman) and [Quantcast](https://github.com/quantcast)! * Added tracer.InjectHeader convenience function for... convenience! Thanks, [mikeh](https://github.com/mikeh-stripe)! * Veneur has a new sink that can be configured to send spans as events into a [Splunk HEC](http://dev.splunk.com/view/event-collector/SP-CAAAE6M) endpoint. Thanks, [antifuchs](https://github.com/antifuchs) and [aditya](https://github.com/chimeracoder)! @@ -167,15 +185,17 @@ * `veneur-emit` will now exit with an error if no data would have been sent. Thanks, [sdboyer](https://github.com/sdboyer)! ## Bugfixes -* The trace client can now correctly parse trace headers emitted by Envoy. Thanks, [aditya](https://github.com/chimeracoder)! +* The trace client can now correctly parse trace headers emitted by Envoy. Thanks, [aditya](https://github.com/chimeracoder)! ## Removed + * Go 1.9 is no longer supported. # 7.0.0, 2018-08-08 ## Added + * `veneur-emit` now takes a new option `-span_tags` for tags that should be applied only to spans. This allows span-specific tags that are not applied to other emitted values. Thanks [gphat](https://github.com/gphat)! * `veneur-emit`'s `-tag` flag now applies the supplied tags to any value emitted, be it a span, metric, service check or event. Use other, mode specific flags (e.g. span_tags) to add tags only to those modes. Thanks [gphat](https://github.com/gphat)! * Isolated a potential resource starvation issue. Added new configuration options for `veneur-proxy` to configure its [http.Transport](https://golang.org/pkg/net/http/#Transport): @@ -188,15 +208,17 @@ * `tracing_client_metrics_interval` for controlling how often thew tracing client will send metrics about it's own operations, defaults to `1s` and is unchanged ## Bugfixes -* `veneur-prometheus` no longer crashes when the metrics host is unreachable. Thanks, [arjenvanderende](https://github.com/arjenvanderende)! +* `veneur-prometheus` no longer crashes when the metrics host is unreachable. Thanks, [arjenvanderende](https://github.com/arjenvanderende)! ## Removed + * `veneur-proxy` now only logs forward counts at Debug level, drastically reducing log volume. # 6.0.0, 2018-06-28 ## Added + * Metrics can be imported over gRPC if the `grpc_address` parameter is set. Thanks, [noahgoldman](https://github.com/noahgoldman) and [Quantcast](https://github.com/quantcast)! * When timing commands, `veneur-emit` now passes stdin, stdout and stderr through to the child process unmodified. Thanks [antifuchs](https://github.com/antifuchs) and [sdboyer](https://github.com/sdboyer)! * Metrics can be forwarded over gRPC (currently only to a single Veneur) using `forward_use_grpc`. Thanks, [noahgoldman](https://github.com/noahgoldman) and [Quantcast](https://github.com/quantcast)! @@ -204,6 +226,7 @@ * `veneur-emit` now takes a new option `-set` with a string argument, which allows counting how many unique values were reported in veneur's flush interval. Thanks, [antifuchs](https://github.com/antifuchs)! ## Bugfixes + * Fix a possible crash-before-panic when unable to open UDP socket. Thanks, [gphat](https://github.com/gphat) * The `StartSpan` method on `tracer.Tracer` will default to the provided `operationName` if provided. This function is provided for compatibility with OpenTracing, but the package-level `trace.StartSpanFromContext` function is recommended for new users. * When creating timer metrics from indicator spans, veneur no longer prefixes `indicator_span_timer_name` with the string `veneur.`. Thanks, [antifuchs](https://github.com/antifuchs)! @@ -211,19 +234,23 @@ * Environment config for `veneur-proxy` now uses `VENEUR_PROXY_` as a prefix. Previously used `VENEUR_` which was a bug! ## Updated + * Metric sampler parse function now looks for `veneurlocalonly` and `veneurglobalonly` by prefix instead of direct equality for times where value can't/shouldn't be excluded even if it's blank. Thanks [joeybloggs](https://github.com/joeybloggs) * `veneur-prometheus` now exports a tag for each quartile rather than a seperate metric ## Removed + * The tag `span_name` has been removed from the timer metric generated for indicator spans. Thanks, [aditya](https://github.com/chimeracoder)! # 5.0.0, 2018-05-17 ## Added + * Added a timeout for sink ingestion to all sinks, which prevents a single slow sink from blocking ingestion on other span sinks indefinitely. Thanks, [aditya](https://github.com/chimeracoder)! * Added `trace.SetDefaultClient` to handle overridding the default trace client, and closing the existing one. Thanks, [franklinhu](https://github.com/franklinhu) ## Improvements + * Veneur's key performance indicator metrics for metrics processing are reported through the statsd client. This way, KPI metrics are affected only by the metrics pipeline, not the tracing pipeline as well. * SignalFX sink can now handle and convert ssf service checks (represented as a gauge). Thanks, [stealthcode](https://github.com/stealthcode)! * Converted the grpsink to use unary instead of stream RPCs. Thanks, [sdboyer](https://github.com/sdboyer)! @@ -237,11 +264,13 @@ * `gc.GCCPUFraction` ## Bugfixes + * The `ignored-labels` and `ignored-metrics` flags for veneur-prometheus will filter no metrics or labels if no filter is specified. Thanks, [arjenvanderende](https://github.com/arjenvanderende)! * Fixed problem where all Datadog service checks were set to `OK` instead of the supplied value. Thanks, [gphat](https://github.com/gphat)! * Added a timeout to the Kafka sink, which prevents the Kafka client from blocking other span sinks. Thanks, [aditya](https://github.com/chimeracoder)! ## Removed + * Official support for building Veneur's binaries with Go 1.8 has been dropped. Supported versions of Go for building Veneur 1.9, 1.10, or tip. * Veneur's trace client library can still be used in applications that are built with Go 1.8, but it is no longer tested against Go 1.8. * The `veneur.ssf.received_total` metric has been removed, as it is mostly redundant with `veneur.ssf.spans.received_total`, and was not reported consistently between packet and framed formats. @@ -250,6 +279,7 @@ # 4.0.0, 2018-04-13 ## Improvements + * Receiving SSF in UDP packets now happens on `num_readers` goroutines. Thanks, [antifuchs](https://github.com/antifuchs) * Updated [SignalFx library](https://github.com/signalfx/golib) dependency so that compression is enabled by default, saving significant time on large metric bodies. Thanks, [gphat](https://github.com/gphat) * Decreased logging output of veneur-proxy. Thanks, [gphat](https://github.com/gphat)! @@ -293,11 +323,13 @@ * Veneur's internal metrics are no longer tagged with `veneurlocalonly`. This means that percentile metrics (such as timers) will now be aggregated globally. ## Bugfixes + * LightStep sink was hardcoded to use plaintext, now adjusts based on URL scheme (http versus https). Thanks [gphat](https://github.com/gphat)! * The Datadog sink will no longer panic if `flush_max_per_body` is not configured; a default is used instead. Thanks [silverlyra](https://github.com/silverlyra)! * The statsd source will no longer reject all packets if `metric_max_length` is not configured; a default is used instead. Thanks [silverlyra](https://github.com/silverlyra)! ## Added + * `veneur-emit` now infers parent and trace IDs from the environment (using the variables `VENEUR_EMIT_TRACE_ID` and `VENEUR_EMIT_PARENT_SPAN_ID`) and sets these environment variables from its `-trace_id` and `parent_span_id` when timing commands, allowing for convenient construction of trace trees if traced programs call `veneur-emit` themselves. Thanks, [antifuchs](https://github.com/antifuchs) * The Kafka sink for spans can now sample spans (at a rate determined by `kafka_span_sample_rate_percent`) based off of traceIDs (by default) or a tag's values (configurable via `kafka_span_sample_tag`) to consistently sample spans related to each other. Thanks, [rhwlo](https://github.com/rhwlo)! * Improvements in SSF metrics reporting - thanks, [antifuchs](https://github.com/antifuchs)! @@ -310,14 +342,17 @@ * Add native support for running Veneur within Kubernetes. Thanks, [aditya](https://github.com/chimeracoder)! ## Improvements + * Updated Datadog span sink to latest version in Datadog tracing agent. Thanks, [gphat](https://github.com/gphat)! # 2.0.0, 2018-01-09 ## Incompatible changes + * The semantics around `veneur-emit` command timing have changed: `-shellCommand` argument has been renamed to `-command`, and `-command` is now gone. The only way to time a command is to provide the command and its arguments as separate arguments, the method of passing in a shell-escaped string is no longer supported. ## Added + * A [SignalFx sink](https://github.com/stripe/veneur/tree/master/sinks/signalfx) has been added for flushing metrics to [SignalFx](https://signalfx.com/). Thanks, [gphat](https://github.com/gphat)! * A [Kafka sink](https://github.com/stripe/veneur/tree/master/sinks/kafka) has been added for publishing spans or metrics. Thanks, [parabuzzle](https://github.com/parabuzzle) and [gphat](https://github.com/gphat)! * Buffered trace clients in `github.com/stripe/veneur/trace` now have a new option to automatically flush them in a periodic interval. Thanks, [antifuchs](https://github.com/antifuchs)! @@ -329,6 +364,7 @@ * Veneur now supports the tag `veneursinkonly:` on metrics, which routes the metric to only the sink specified. See [the docs](README.md#routing-metrics) here. Thanks, [antifuchs](https://github.com/antifuchs)! ## Improvements + * Veneur now emits a timer metric giving the duration (in nanoseconds) of every "indicator" span that it receives, if you configure the setting `indicator_span_timer_name`. Thanks, [antifuchs](https://github.com/antifuchs)! * All sinks have been moved to their own packages for smaller code and better interfaces. Thanks [gphat](https://github.com/gphat)! * Removed noisy Sentry events that duplicated Datadog error reporting. Thanks, [aditya](https://github.com/chimeracoder)! @@ -337,14 +373,17 @@ # 1.8.1, 2017-12-05 ## Improvements + * Veneur now tracks statsd metrics for SSF spans concerning its own operation. This means that the `veneur.ssf.spans.received_total` counter and the `veneur.ssf.packet_size` histogram again reflect trace spans routed internally. Thanks, [antifuchs](https://github.com/antifuchs)! # 1.8.0, 2017-11-29 ## Added + * New 'blackhole' sink for testing and benchmark purposes. Thanks [gphat](https://github.com/gphat)! ## Improvements + * Veneur no longer **requires** the use of Datadog as a target for flushes. Veneur can now use one or more of any of its supported sinks as a backend. This realizes our desire for Veneur to be fully vendor agnostic. Thanks [gphat](https://github.com/gphat)! * The package `github.com/stripe/veneur/trace` now depends on fewer other packages across veneur, making it easier to pull in `trace` as a dependency. Thanks [antifuchs](https://github.com/antifuchs)! * A Veneur server with tracing enabled now submits traces and spans concerning its own operation to itself internally without sending them over UDP. See the "Upgrade Notes" section below for metrics affected by this change. Thanks [antifuchs](https://github.com/antifuchs)! @@ -357,19 +396,23 @@ * Refactor internal HTTP helper into its own package fix up possible circular deps. Thanks [gphat](https://github.com/gphat)! ## Bugfixes + * Fix a panic when using `veneur-emit` to emit metrics via `-ssf` when no tags are specified. Thanks [myndzi](https://github.com/myndzi) * Remove spurious warnings about unset configuration settings. Thanks [antifuchs](https://github.com/antifuchs) ## Removed + * Removed the InfluxDB plugin as it was experimental and wasn't working. We can make a sink for it in the future if desired. Thanks [gphat](https://github.com/gphat)! # 1.7.0, 2017-10-19 ## Notes for upgrading from previous versions + * The `set` data structure serialization format for communiation with a global Veneur server has changed in an incompatible way. If your infrastructure relies on a global Veneur installation, they will drop `set` data from non-matching versions until the entire fleet and the global Veneur are all at the same version. * The metrics for SSF packets (and spans) received have changed names: They used to be `veneur.packet.received_total` and `veneur.packet.spans.received_total`, respectively, and they are now named `veneur.ssf.received_total` and `veneur.ssf.spans.received_total`. ## Added + * New [configuration option](https://github.com/stripe/veneur/pull/233) `statsd_listen_addresses`, a list of URIs indicating on which ports (and protocols) Veneur should listen on for statsd metrics. This deprecates both the `udp_address` and `tcp_address` settings. Thanks [antifuchs](https://github.com/antifuchs)! * New package `github.com/stripe/veneur/protocol`, containing a wire protocol for sending/reading SSF over a streaming connection. Thanks [antifuchs](https://github.com/antifuchs)! * `github.com/veneur/trace` now contains [customizable `Client`s](https://github.com/stripe/veneur/pull/262) that support streaming connections. @@ -380,16 +423,19 @@ * The BUILD_DATE and VERSION variables can be set at link-time and are now exposed by the `/builddate` and `/version` endpoints. ## Improvements + * [A new HyperLogLog implementation](https://github.com/stripe/veneur/pull/190) means `set`s are faster and allocate less memory. Thanks, [martinpinto](https://github.com/martinpinto) and [seiflotfy](https://github.com/seiflotfy)! * Introduced a new `metricSink` which provides a common interface for metric backends. In an upcoming release all plugins will be converted to this interface. Thanks [gphat](https://github.com/gphat)! ## Deprecations + * `veneur-emit` no longer supports the `-config` argument, as it's no longer possible to reliably detect which statsd host/port to connect to. The `-hostport` option now takes a URL of the same form `statsd_listen_addresses` takes to explicitly tell it what address it should send to. * `SSFSpanCollection` got removed, as it is superseded by the wire protocol. If you need to send multiple spans in bulk, we recommend setting up a buffered `trace.Client`! # 1.6.0, 2017-08-29 ## Added + * Veneur-emit [can now time any shell command](https://github.com/stripe/veneur/pull/222) and emit its duration as a Timing metric. Thanks [redsn0w422](https://github.com/redsn0w422)! * Config options can now be provided via environment variables using [envconfig](https://github.com/kelseyhightower/envconfig) for Veneur and veneur-proxy. Thanks [gphat](https://github.com/gphat)! * [SSF](https://github.com/stripe/veneur/tree/master/ssf) now includes a boolean `indicator` field for signaling that this span is useful as a [Service Level Indicator](https://en.wikipedia.org/wiki/Service_level_indicator) for its service. @@ -411,21 +457,25 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be * Use [gogo protobuf](https://github.com/gogo/protobuf) for [code generation of SSF's protobuf](https://github.com/stripe/veneur/pull/236), resulting in faster and less memory span ingestion. Thanks [gphat](https://github.com/gphat)! ## Bugfixes + * veneur-proxy no longer balks at using static hosts for tracing and metrics. Thanks [gphat](https://github.com/gphat)! ## Deprecations + * SSF's `operation` field has been deprecated in favor of the field `name`. * SSF spans with a tag `name` will have that name placed into the SSF span `name` field until 2.0 is released. # 1.5.2, 2017-08-15 ## Bugfixes + * Correctly parse `Set` metrics if sent via SSF. Thanks [redsn0w422](https://github.com/redsn0w422)! * Return correct array of tags after parsing an SSF metric. Thanks [redsn0w422](https://github.com/redsn0w422)! * Don't panic if a packet doesn't have tags. Thanks [redsn0w422](https://github.com/redsn0w422)! * Fix a typo in the link to veneur-emit in the readme. Thanks [vasi](https://github.com/vasi)! ## Added + * Adds [events](https://docs.datadoghq.com/guides/dogstatsd/#events-1) and [service checks](https://docs.datadoghq.com/guides/dogstatsd/#service-checks-1) to `veneur-emit`. Thanks [redsn0w422](https://github.com/redsn0w422)! * Switch to [dep](https://github.com/golang/dep/) for managing the `vendor` directory. Thanks [chimeracoder](https://github.com/chimeracoder)! * Remove support for `govendor`. Thanks [chimeracoder](https://github.com/chimeracoder)! @@ -436,6 +486,7 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be * Added [harmonic mean](https://en.wikipedia.org/wiki/Harmonic_mean) as an optional aggregate type. Use `hmean` as an option to the `aggregates` config option to enable. Thanks [non](https://github.com/non)! ## Improvements + * Added tests for `parseMetricSSF`. Thanks [redsn0w422](https://github.com/redsn0w422)! * Refactored `veneur-emit` flag usage to make testing easier. Thanks [redsn0w422](https://github.com/redsn0w422)! * Minor text fixes in the README. Thanks [an-stripe](https://github.com/an-stripe)! @@ -445,24 +496,29 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be # 1.5.1, 2017-07-18 ## Bugfixes + * Flush the lightstep tracer before closing it. Thanks [gphat](https://github.com/gphat) with assist from [stangles](https://github.com/joshu-stripe)! ## Improvements + * Better document how to configure Veneur as a DogStatsD replacement. Thanks [gphat](https://github.com/gphat) with assist from [stangles](https://github.com/stangles)! # 1.5.0, 2017-07-13 ## Bugfixes + * Fixed an error in graceful shutdown of the TCP listener. Thanks [evanj](https://github.com/evanj)! * Don't hang if we call `log.Fatal` and we aren't hooked up to a Sentry. Thanks [evanj](https://github.com/evanj)! * Fix flusher_test being called more than once resulting in flappy failure. Thanks [evanj](https://github.com/evanj)! * Improve flusher test to not start Veneur, fixing flapping test. Thanks [evanj](https://github.com/evanj)! ## Added + * `veneur-emit` can now emit metrics using the [SSF protocol](https://github.com/stripe/veneur/tree/master/ssf#readme). Thanks [redsn0w422](https://github.com/redsn0w422)! * Documentation for SSF. Thanks [gphat](https://github.com/gphat)! ## Improvements + * It is no longer required to emit a `sum` to get an `avg` when configuring what aggregations to emit for a histogram. Thanks [cgilling](https://github.com/cgilling)! * Tags added in the `tags` config key are now applied to trace spans. Thanks [chimeracoder](https://github.com/chimeracoder)! * Additional documentation for `veneur-proxy`. Thanks [gphat](https://github.com/gphat)! @@ -473,6 +529,7 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be * Reresolve the LightStep trace flusher on each flush, accomodating Consul-based DNS use and preventing stale sinks. Thanks [chimeracoder](https://github.com/chimeracoder)! ## Deprecations + * The following configuration keys are deprecated and will be removed in version 2.0 of Veneur: * `datadog_api_key` replaces `key` * `datadog_api_hostname` replaces `api_hostname` @@ -482,33 +539,40 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be # 1.4.0, 2017-06-09 ## Changes + * Require Go 1.8+ and stop building against 1.7 Thanks Thanks [chimeracoder](https://github.com/chimeracoder)! # 1.3.1, 2017-06-06 ## Bugfixes + * Decrease logging level for proxy's "forwarded" messages. Thanks [gphat](https://github.com/gphat)! * Failed discovery refreshes now log the service name. Thanks [gphat](https://github.com/gphat)! ## Improvements + * Proxy no longer requires a trace service name, since it's not wired up. Thanks [gphat](https://github.com/gphat)! # 1.3.0, 2017-05-19 ## Bugfixes + * No longer allow clients to pass in `nan`, `+inf` or `-inf` as a value for a metric, as this caused errors on flush. Thanks [gphat](https://github.com/gphat)! ## Added + * Added `veneur-proxy` to provide HA features with consistent hashing. See the [Proxy section of the README](https://github.com/stripe/veneur#proxy) # 1.2.0, 2017-04-24 ## Bugfixes + * Fix flusher_test to properly shutdown HTTP after handling. Thanks [evanj](https://github.com/evanj)! * Verify that `trace_max_length_bytes` is properly set. Thanks [evanj](https://github.com/evanj)! * Fix some race conditions in testing. ## Improvements + * Document performance cost of TLS with RSA and ECDH keys. Thanks [evanj](https://github.com/evanj)! * Reduce logging of tracing information to `debug` level to decrease unnecessary logging. * Reduce common TCP error logs to `info` level. Thanks [evanj](https://github.com/evanj)! @@ -521,6 +585,7 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be * Fixed a lot of go lint errors. ## Added + * Add a metric `veneur.sentry.errors_total` for number of errors we send to Sentry. * New plugin `flush_file` for writing metrics to a flat file. * New `/healthcheck/tracing` endpoint that returns 200 if this Veneur instance is accepting traces. @@ -528,15 +593,18 @@ Secondly, the shift in *not* buffering spans on their way to LightStep should be # 1.1.0, 2017-03-02 ## Changes + * Refactor tests to use a more shareable test fixture. Thanks [evanj](https://github.com/evanj)! * Refactor `Server`'s constructor to not start any goroutines and add a `Start()` that takes care of that, making for easier tests. ## Bugfixes + * Hostname and device name tags are now omitted from JSON generated for transmission to Datadog at flush time. Thanks [evanj](https://github.com/evanj)! * Fix panic when an error is generated and Sentry is not configured. Thanks [evanj](https://github.com/evanj)! * Fix typos in README ## Improvements + * Add `omit_empty_hostname` option. If true and `hostname` tag is set to empty, Veneur will not add a host tag to its own metrics. Thanks [evanj](https://github.com/evanj)! * Support "all interfaces" addresses (`:1234`) for listening configuration. Thanks [evanj](https://github.com/evanj)! * Add support for receiving statsd packets over authenticated TLS connections. Thanks [evanj](https://github.com/evanj)! From 7eb43181700ce407051d2aae65f25c8428e506f5 Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 12:41:28 +0200 Subject: [PATCH 3/8] Fix typo in README.md --- public-docker-images/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public-docker-images/README.md b/public-docker-images/README.md index 41dc4fcd5..889f4d58e 100644 --- a/public-docker-images/README.md +++ b/public-docker-images/README.md @@ -9,13 +9,13 @@ You should run these commands from the project root. For the Debian-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur-local -f public-docker-images/Dockerfile-debian-sid --pull --push . +docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --pull --push . ``` For the Alpine Linux-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur-local -f public-docker-images/Dockerfile-alpine --pull --push . +docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --pull --push . ``` For both cases you could remove ```--platform` arugment if you just plan build for the host architechture. From 1463c78f8c5ba7755504b27d18932fc253f902f3 Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 12:44:49 +0200 Subject: [PATCH 4/8] Fix the PR number on changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccbc115ab..cc92e893a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ * Use `T.TempDir` to create temporary directory in tests ([#944](https://github.com/stripe/veneur/pull/944)). * When the request to send data from Cloudwatch & SFX sink fails, log the count of metrics that are dropped. -* Improve multi-arch build by making use golang cross-compilation ([#999](https://github.com/stripe/veneur/pull/990)) +* Improve multi-arch build by making use golang cross-compilation ([#990](https://github.com/stripe/veneur/pull/990)) ## Bugfixes From 3d9c5be25eb4ae485de5f06eaef9b9c14367a2fc Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 12:47:48 +0200 Subject: [PATCH 5/8] Re-add build-arg I accidentally removed --- public-docker-images/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public-docker-images/README.md b/public-docker-images/README.md index 889f4d58e..c1de4a036 100644 --- a/public-docker-images/README.md +++ b/public-docker-images/README.md @@ -9,13 +9,13 @@ You should run these commands from the project root. For the Debian-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --pull --push . +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --pull --push . ``` For the Alpine Linux-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --pull --push . +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --pull --push . ``` For both cases you could remove ```--platform` arugment if you just plan build for the host architechture. @@ -49,5 +49,5 @@ $ docker run --rm -it \ `veneur-emit` is also included in the image, and can be run explicitly like so: ``` -docker run --net=host -it veneur /veneur/veneur-emit -count 1 -name foo -hostport "127.0.0.1:8126" +$ docker run --net=host -it veneur /veneur/veneur-emit -count 1 -name foo -hostport "127.0.0.1:8126" ``` From d02685496be0aa929d2241c62d426c323de11ffd Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 13:02:31 +0200 Subject: [PATCH 6/8] Update readme to have the generated image available on Mac --- public-docker-images/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public-docker-images/README.md b/public-docker-images/README.md index c1de4a036..f8691d25f 100644 --- a/public-docker-images/README.md +++ b/public-docker-images/README.md @@ -9,13 +9,13 @@ You should run these commands from the project root. For the Debian-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --pull --push . +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --output tar . | docker import - ``` For the Alpine Linux-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --pull --push . +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --output tar . | docker import - ``` For both cases you could remove ```--platform` arugment if you just plan build for the host architechture. @@ -49,5 +49,5 @@ $ docker run --rm -it \ `veneur-emit` is also included in the image, and can be run explicitly like so: ``` -$ docker run --net=host -it veneur /veneur/veneur-emit -count 1 -name foo -hostport "127.0.0.1:8126" +docker run --net=host -it veneur /veneur/veneur-emit -count 1 -name foo -hostport "127.0.0.1:8126" ``` From 7608d76165a5d7e8c2d5cbe17783f3c2e2d3c64b Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 12 Sep 2022 13:07:44 +0200 Subject: [PATCH 7/8] Re-add extra empty line on Dockerfile-alpine --- public-docker-images/Dockerfile-alpine | 1 + public-docker-images/README.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/public-docker-images/Dockerfile-alpine b/public-docker-images/Dockerfile-alpine index 384f185f2..691f0aa79 100644 --- a/public-docker-images/Dockerfile-alpine +++ b/public-docker-images/Dockerfile-alpine @@ -7,6 +7,7 @@ RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -ldflags "-X git GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -o /build/veneur-prometheus ./cmd/veneur-prometheus &&\ GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -a -ldflags "-X github.com/stripe/veneur/v14/util/build.VERSION=${VERSION} -X github.com/stripe/veneur/v14/util/build.BUILD_DATE=$(date +%s)" -o /build/veneur-proxy ./cmd/veneur-proxy + FROM alpine:3.16 AS release LABEL maintainer="The Stripe Observability Team " RUN apk add --no-cache ca-certificates diff --git a/public-docker-images/README.md b/public-docker-images/README.md index f8691d25f..73285a00d 100644 --- a/public-docker-images/README.md +++ b/public-docker-images/README.md @@ -9,13 +9,13 @@ You should run these commands from the project root. For the Debian-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --output tar . | docker import - +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --output=tar . | docker import - ``` For the Alpine Linux-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --output tar . | docker import - +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --output=tar . | docker import - ``` For both cases you could remove ```--platform` arugment if you just plan build for the host architechture. From e06d88c5af8c1409e2e5c0353358340d4c43cbbb Mon Sep 17 00:00:00 2001 From: Rafael Gaspar Date: Mon, 19 Sep 2022 14:53:13 +0200 Subject: [PATCH 8/8] Improve documentation with both single and multi-arch steps --- public-docker-images/README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/public-docker-images/README.md b/public-docker-images/README.md index 73285a00d..be6665655 100644 --- a/public-docker-images/README.md +++ b/public-docker-images/README.md @@ -9,16 +9,32 @@ You should run these commands from the project root. For the Debian-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --output=tar . | docker import - +docker build --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid . ``` For the Alpine Linux-based image: ``` -docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --output=tar . | docker import - +docker build --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine . ``` -For both cases you could remove ```--platform` arugment if you just plan build for the host architechture. +## Building with multi-arch + +You should run these commands from the project root. + +For the Debian-based image: + +``` +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-debian-sid --output=type=docker +``` + +For the Alpine Linux-based image: + +``` +docker buildx build --platform=linux/amd64,linux/arm64 --build-arg=VERSION=$(git rev-parse HEAD) --no-cache -t veneur:local -f public-docker-images/Dockerfile-alpine --output=type=docker +``` + +Multi-arch build works best when pushing to a remote repository, so tag it accordingly and replace `--output=type=docker` with `--push` for that to work. ## Running