GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

cloudflare/cloudflared

Public

mirrored from https://github.com/cloudflare/cloudflaredAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
2020.11.7

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

cloudflared/connection

connection/metrics.go

405lines · modecode

RawDownload
Latest commit unavailable.
1package connection
2
3import (
4 "sync"
5 "time"
6
7 "github.com/cloudflare/cloudflared/h2mux"
8 "github.com/prometheus/client_golang/prometheus"
9)
10
11const (
12 MetricsNamespace = "cloudflared"
13 TunnelSubsystem = "tunnel"
14 muxerSubsystem = "muxer"
15)
16
17type muxerMetrics struct {
18 rtt *prometheus.GaugeVec
19 rttMin *prometheus.GaugeVec
20 rttMax *prometheus.GaugeVec
21 receiveWindowAve *prometheus.GaugeVec
22 sendWindowAve *prometheus.GaugeVec
23 receiveWindowMin *prometheus.GaugeVec
24 receiveWindowMax *prometheus.GaugeVec
25 sendWindowMin *prometheus.GaugeVec
26 sendWindowMax *prometheus.GaugeVec
27 inBoundRateCurr *prometheus.GaugeVec
28 inBoundRateMin *prometheus.GaugeVec
29 inBoundRateMax *prometheus.GaugeVec
30 outBoundRateCurr *prometheus.GaugeVec
31 outBoundRateMin *prometheus.GaugeVec
32 outBoundRateMax *prometheus.GaugeVec
33 compBytesBefore *prometheus.GaugeVec
34 compBytesAfter *prometheus.GaugeVec
35 compRateAve *prometheus.GaugeVec
36}
37
38type tunnelMetrics struct {
39 timerRetries prometheus.Gauge
40 serverLocations *prometheus.GaugeVec
41 // locationLock is a mutex for oldServerLocations
42 locationLock sync.Mutex
43 // oldServerLocations stores the last server the tunnel was connected to
44 oldServerLocations map[string]string
45
46 regSuccess *prometheus.CounterVec
47 regFail *prometheus.CounterVec
48 rpcFail *prometheus.CounterVec
49
50 muxerMetrics *muxerMetrics
51 tunnelsHA tunnelsForHA
52 userHostnamesCounts *prometheus.CounterVec
53}
54
55func newMuxerMetrics() *muxerMetrics {
56 rtt := prometheus.NewGaugeVec(
57 prometheus.GaugeOpts{
58 Namespace: MetricsNamespace,
59 Subsystem: muxerSubsystem,
60 Name: "rtt",
61 Help: "Round-trip time in millisecond",
62 },
63 []string{"connection_id"},
64 )
65 prometheus.MustRegister(rtt)
66
67 rttMin := prometheus.NewGaugeVec(
68 prometheus.GaugeOpts{
69 Namespace: MetricsNamespace,
70 Subsystem: muxerSubsystem,
71 Name: "rtt_min",
72 Help: "Shortest round-trip time in millisecond",
73 },
74 []string{"connection_id"},
75 )
76 prometheus.MustRegister(rttMin)
77
78 rttMax := prometheus.NewGaugeVec(
79 prometheus.GaugeOpts{
80 Namespace: MetricsNamespace,
81 Subsystem: muxerSubsystem,
82 Name: "rtt_max",
83 Help: "Longest round-trip time in millisecond",
84 },
85 []string{"connection_id"},
86 )
87 prometheus.MustRegister(rttMax)
88
89 receiveWindowAve := prometheus.NewGaugeVec(
90 prometheus.GaugeOpts{
91 Namespace: MetricsNamespace,
92 Subsystem: muxerSubsystem,
93 Name: "receive_window_ave",
94 Help: "Average receive window size in bytes",
95 },
96 []string{"connection_id"},
97 )
98 prometheus.MustRegister(receiveWindowAve)
99
100 sendWindowAve := prometheus.NewGaugeVec(
101 prometheus.GaugeOpts{
102 Namespace: MetricsNamespace,
103 Subsystem: muxerSubsystem,
104 Name: "send_window_ave",
105 Help: "Average send window size in bytes",
106 },
107 []string{"connection_id"},
108 )
109 prometheus.MustRegister(sendWindowAve)
110
111 receiveWindowMin := prometheus.NewGaugeVec(
112 prometheus.GaugeOpts{
113 Namespace: MetricsNamespace,
114 Subsystem: muxerSubsystem,
115 Name: "receive_window_min",
116 Help: "Smallest receive window size in bytes",
117 },
118 []string{"connection_id"},
119 )
120 prometheus.MustRegister(receiveWindowMin)
121
122 receiveWindowMax := prometheus.NewGaugeVec(
123 prometheus.GaugeOpts{
124 Namespace: MetricsNamespace,
125 Subsystem: muxerSubsystem,
126 Name: "receive_window_max",
127 Help: "Largest receive window size in bytes",
128 },
129 []string{"connection_id"},
130 )
131 prometheus.MustRegister(receiveWindowMax)
132
133 sendWindowMin := prometheus.NewGaugeVec(
134 prometheus.GaugeOpts{
135 Namespace: MetricsNamespace,
136 Subsystem: muxerSubsystem,
137 Name: "send_window_min",
138 Help: "Smallest send window size in bytes",
139 },
140 []string{"connection_id"},
141 )
142 prometheus.MustRegister(sendWindowMin)
143
144 sendWindowMax := prometheus.NewGaugeVec(
145 prometheus.GaugeOpts{
146 Namespace: MetricsNamespace,
147 Subsystem: muxerSubsystem,
148 Name: "send_window_max",
149 Help: "Largest send window size in bytes",
150 },
151 []string{"connection_id"},
152 )
153 prometheus.MustRegister(sendWindowMax)
154
155 inBoundRateCurr := prometheus.NewGaugeVec(
156 prometheus.GaugeOpts{
157 Namespace: MetricsNamespace,
158 Subsystem: muxerSubsystem,
159 Name: "inbound_bytes_per_sec_curr",
160 Help: "Current inbounding bytes per second, 0 if there is no incoming connection",
161 },
162 []string{"connection_id"},
163 )
164 prometheus.MustRegister(inBoundRateCurr)
165
166 inBoundRateMin := prometheus.NewGaugeVec(
167 prometheus.GaugeOpts{
168 Namespace: MetricsNamespace,
169 Subsystem: muxerSubsystem,
170 Name: "inbound_bytes_per_sec_min",
171 Help: "Minimum non-zero inbounding bytes per second",
172 },
173 []string{"connection_id"},
174 )
175 prometheus.MustRegister(inBoundRateMin)
176
177 inBoundRateMax := prometheus.NewGaugeVec(
178 prometheus.GaugeOpts{
179 Namespace: MetricsNamespace,
180 Subsystem: muxerSubsystem,
181 Name: "inbound_bytes_per_sec_max",
182 Help: "Maximum inbounding bytes per second",
183 },
184 []string{"connection_id"},
185 )
186 prometheus.MustRegister(inBoundRateMax)
187
188 outBoundRateCurr := prometheus.NewGaugeVec(
189 prometheus.GaugeOpts{
190 Namespace: MetricsNamespace,
191 Subsystem: muxerSubsystem,
192 Name: "outbound_bytes_per_sec_curr",
193 Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic",
194 },
195 []string{"connection_id"},
196 )
197 prometheus.MustRegister(outBoundRateCurr)
198
199 outBoundRateMin := prometheus.NewGaugeVec(
200 prometheus.GaugeOpts{
201 Namespace: MetricsNamespace,
202 Subsystem: muxerSubsystem,
203 Name: "outbound_bytes_per_sec_min",
204 Help: "Minimum non-zero outbounding bytes per second",
205 },
206 []string{"connection_id"},
207 )
208 prometheus.MustRegister(outBoundRateMin)
209
210 outBoundRateMax := prometheus.NewGaugeVec(
211 prometheus.GaugeOpts{
212 Namespace: MetricsNamespace,
213 Subsystem: muxerSubsystem,
214 Name: "outbound_bytes_per_sec_max",
215 Help: "Maximum outbounding bytes per second",
216 },
217 []string{"connection_id"},
218 )
219 prometheus.MustRegister(outBoundRateMax)
220
221 compBytesBefore := prometheus.NewGaugeVec(
222 prometheus.GaugeOpts{
223 Namespace: MetricsNamespace,
224 Subsystem: muxerSubsystem,
225 Name: "comp_bytes_before",
226 Help: "Bytes sent via cross-stream compression, pre compression",
227 },
228 []string{"connection_id"},
229 )
230 prometheus.MustRegister(compBytesBefore)
231
232 compBytesAfter := prometheus.NewGaugeVec(
233 prometheus.GaugeOpts{
234 Namespace: MetricsNamespace,
235 Subsystem: muxerSubsystem,
236 Name: "comp_bytes_after",
237 Help: "Bytes sent via cross-stream compression, post compression",
238 },
239 []string{"connection_id"},
240 )
241 prometheus.MustRegister(compBytesAfter)
242
243 compRateAve := prometheus.NewGaugeVec(
244 prometheus.GaugeOpts{
245 Namespace: MetricsNamespace,
246 Subsystem: muxerSubsystem,
247 Name: "comp_rate_ave",
248 Help: "Average outbound cross-stream compression ratio",
249 },
250 []string{"connection_id"},
251 )
252 prometheus.MustRegister(compRateAve)
253
254 return &muxerMetrics{
255 rtt: rtt,
256 rttMin: rttMin,
257 rttMax: rttMax,
258 receiveWindowAve: receiveWindowAve,
259 sendWindowAve: sendWindowAve,
260 receiveWindowMin: receiveWindowMin,
261 receiveWindowMax: receiveWindowMax,
262 sendWindowMin: sendWindowMin,
263 sendWindowMax: sendWindowMax,
264 inBoundRateCurr: inBoundRateCurr,
265 inBoundRateMin: inBoundRateMin,
266 inBoundRateMax: inBoundRateMax,
267 outBoundRateCurr: outBoundRateCurr,
268 outBoundRateMin: outBoundRateMin,
269 outBoundRateMax: outBoundRateMax,
270 compBytesBefore: compBytesBefore,
271 compBytesAfter: compBytesAfter,
272 compRateAve: compRateAve,
273 }
274}
275
276func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
277 m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
278 m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
279 m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
280 m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
281 m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
282 m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
283 m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
284 m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
285 m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
286 m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
287 m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
288 m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
289 m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
290 m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
291 m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
292 m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
293 m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
294 m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
295}
296
297func convertRTTMilliSec(t time.Duration) float64 {
298 return float64(t / time.Millisecond)
299}
300
301// Metrics that can be collected without asking the edge
302func newTunnelMetrics() *tunnelMetrics {
303 maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
304 prometheus.GaugeOpts{
305 Namespace: MetricsNamespace,
306 Subsystem: TunnelSubsystem,
307 Name: "max_concurrent_requests_per_tunnel",
308 Help: "Largest number of concurrent requests proxied through each tunnel so far",
309 },
310 []string{"connection_id"},
311 )
312 prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
313
314 timerRetries := prometheus.NewGauge(
315 prometheus.GaugeOpts{
316 Namespace: MetricsNamespace,
317 Subsystem: TunnelSubsystem,
318 Name: "timer_retries",
319 Help: "Unacknowledged heart beats count",
320 })
321 prometheus.MustRegister(timerRetries)
322
323 serverLocations := prometheus.NewGaugeVec(
324 prometheus.GaugeOpts{
325 Namespace: MetricsNamespace,
326 Subsystem: TunnelSubsystem,
327 Name: "server_locations",
328 Help: "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
329 },
330 []string{"connection_id", "location"},
331 )
332 prometheus.MustRegister(serverLocations)
333
334 rpcFail := prometheus.NewCounterVec(
335 prometheus.CounterOpts{
336 Namespace: MetricsNamespace,
337 Subsystem: TunnelSubsystem,
338 Name: "tunnel_rpc_fail",
339 Help: "Count of RPC connection errors by type",
340 },
341 []string{"error", "rpcName"},
342 )
343 prometheus.MustRegister(rpcFail)
344
345 registerFail := prometheus.NewCounterVec(
346 prometheus.CounterOpts{
347 Namespace: MetricsNamespace,
348 Subsystem: TunnelSubsystem,
349 Name: "tunnel_register_fail",
350 Help: "Count of tunnel registration errors by type",
351 },
352 []string{"error", "rpcName"},
353 )
354 prometheus.MustRegister(registerFail)
355
356 userHostnamesCounts := prometheus.NewCounterVec(
357 prometheus.CounterOpts{
358 Namespace: MetricsNamespace,
359 Subsystem: TunnelSubsystem,
360 Name: "user_hostnames_counts",
361 Help: "Which user hostnames cloudflared is serving",
362 },
363 []string{"userHostname"},
364 )
365 prometheus.MustRegister(userHostnamesCounts)
366
367 registerSuccess := prometheus.NewCounterVec(
368 prometheus.CounterOpts{
369 Namespace: MetricsNamespace,
370 Subsystem: TunnelSubsystem,
371 Name: "tunnel_register_success",
372 Help: "Count of successful tunnel registrations",
373 },
374 []string{"rpcName"},
375 )
376 prometheus.MustRegister(registerSuccess)
377
378 return &tunnelMetrics{
379 timerRetries: timerRetries,
380 serverLocations: serverLocations,
381 oldServerLocations: make(map[string]string),
382 muxerMetrics: newMuxerMetrics(),
383 tunnelsHA: NewTunnelsForHA(),
384 regSuccess: registerSuccess,
385 regFail: registerFail,
386 rpcFail: rpcFail,
387 userHostnamesCounts: userHostnamesCounts,
388 }
389}
390
391func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
392 t.muxerMetrics.update(connectionID, metrics)
393}
394
395func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
396 t.locationLock.Lock()
397 defer t.locationLock.Unlock()
398 if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
399 return
400 } else if ok {
401 t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
402 }
403 t.serverLocations.WithLabelValues(connectionID, loc).Inc()
404 t.oldServerLocations[connectionID] = loc
405}
406