From 1a807cce3586a7b4c994e8e55555a494fa0dfac1 Mon Sep 17 00:00:00 2001 From: Aditya Mukerjee Date: Mon, 16 Oct 2017 17:13:42 -0400 Subject: [PATCH 1/3] Add metrics client to tracing client --- trace/client.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/trace/client.go b/trace/client.go index 5e8054894..834baea36 100644 --- a/trace/client.go +++ b/trace/client.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "sync" "time" "github.com/stripe/veneur/protocol" @@ -36,6 +37,9 @@ type Client struct { backend cancel context.CancelFunc ops chan op + + stats IncrClient + statsMtx sync.Mutex } // Close tears down the entire client. It waits until the backend has @@ -61,6 +65,18 @@ func (c *Client) run(ctx context.Context) { } } +type IncrClient interface { + Incr(name string, tags []string, rate float64) error +} + +// SetStats sets an IncrClient used for reporting errors +func (c *Client) SetErrorStats(statsClient IncrClient) { + c.statsMtx.Lock() + defer c.statsMtx.Unlock() + + c.stats = statsClient +} + // ClientParam is an option for NewClient. Its implementation borrows // from Dave Cheney's functional options API // (https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis). From d74545c3e352d72befacaea5dc7c75b7ced41a66 Mon Sep 17 00:00:00 2001 From: Aditya Mukerjee Date: Mon, 16 Oct 2017 17:21:15 -0400 Subject: [PATCH 2/3] Track errors when recording spans --- trace/trace.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/trace/trace.go b/trace/trace.go index c3d8e25cc..835f9e9ce 100644 --- a/trace/trace.go +++ b/trace/trace.go @@ -8,6 +8,7 @@ package trace import ( "context" + "fmt" "io" "math/rand" "reflect" @@ -194,7 +195,14 @@ func (t *Trace) ClientRecord(cl *Client, name string, tags map[string]string) er span := t.SSFSpan() span.Name = name - return Record(cl, span, t.Sent) + err := Record(cl, span, t.Sent) + if err != nil { + cl.statsMtx.Lock() + defer cl.statsMtx.Unlock() + tags := []string{fmt.Sprintf("error:%s", err.Error())} + cl.stats.Incr("veneur.trace.record.errors_total", tags, 0.1) + } + return err } func (t *Trace) Error(err error) { From 72114cab92902b8934314269ffa727e7371085bd Mon Sep 17 00:00:00 2001 From: Aditya Mukerjee Date: Mon, 16 Oct 2017 17:23:43 -0400 Subject: [PATCH 3/3] Set statsd client for reporting errors in server initialization --- server.go | 1 + trace/client.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index 2d2c783d9..b9950f964 100644 --- a/server.go +++ b/server.go @@ -158,6 +158,7 @@ func NewFromConfig(conf Config) (ret Server, err error) { } ret.Statsd.Namespace = "veneur." ret.Statsd.Tags = append(ret.Tags, "veneurlocalonly") + trace.DefaultClient.SetErrorStats(ret.Statsd) // nil is a valid sentry client that noops all methods, if there is no DSN // we can just leave it as nil diff --git a/trace/client.go b/trace/client.go index 834baea36..1c0db4e64 100644 --- a/trace/client.go +++ b/trace/client.go @@ -69,7 +69,7 @@ type IncrClient interface { Incr(name string, tags []string, rate float64) error } -// SetStats sets an IncrClient used for reporting errors +// SetErrorStats sets an IncrClient used for reporting errors func (c *Client) SetErrorStats(statsClient IncrClient) { c.statsMtx.Lock() defer c.statsMtx.Unlock()