GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

cloudflare/cloudflared

Public

mirrored from https://github.com/cloudflare/cloudflaredAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
2018.12.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

cloudflared/origin

origin/metrics.go

423lines · modecode

RawDownload
Latest commit unavailable.
1package origin
2
3import (
4 "sync"
5 "time"
6
7 "github.com/cloudflare/cloudflared/h2mux"
8
9 "github.com/prometheus/client_golang/prometheus"
10)
11
12type muxerMetrics struct {
13 rtt *prometheus.GaugeVec
14 rttMin *prometheus.GaugeVec
15 rttMax *prometheus.GaugeVec
16 receiveWindowAve *prometheus.GaugeVec
17 sendWindowAve *prometheus.GaugeVec
18 receiveWindowMin *prometheus.GaugeVec
19 receiveWindowMax *prometheus.GaugeVec
20 sendWindowMin *prometheus.GaugeVec
21 sendWindowMax *prometheus.GaugeVec
22 inBoundRateCurr *prometheus.GaugeVec
23 inBoundRateMin *prometheus.GaugeVec
24 inBoundRateMax *prometheus.GaugeVec
25 outBoundRateCurr *prometheus.GaugeVec
26 outBoundRateMin *prometheus.GaugeVec
27 outBoundRateMax *prometheus.GaugeVec
28 compBytesBefore *prometheus.GaugeVec
29 compBytesAfter *prometheus.GaugeVec
30 compRateAve *prometheus.GaugeVec
31}
32
33type TunnelMetrics struct {
34 haConnections prometheus.Gauge
35 totalRequests prometheus.Counter
36 requestsPerTunnel *prometheus.CounterVec
37 // concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
38 concurrentRequestsLock sync.Mutex
39 concurrentRequestsPerTunnel *prometheus.GaugeVec
40 // concurrentRequests records count of concurrent requests for each tunnel
41 concurrentRequests map[string]uint64
42 maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
43 // concurrentRequests records max count of concurrent requests for each tunnel
44 maxConcurrentRequests map[string]uint64
45 timerRetries prometheus.Gauge
46 responseByCode *prometheus.CounterVec
47 responseCodePerTunnel *prometheus.CounterVec
48 serverLocations *prometheus.GaugeVec
49 // locationLock is a mutex for oldServerLocations
50 locationLock sync.Mutex
51 // oldServerLocations stores the last server the tunnel was connected to
52 oldServerLocations map[string]string
53
54 muxerMetrics *muxerMetrics
55 tunnelsHA tunnelsForHA
56}
57
58func newMuxerMetrics() *muxerMetrics {
59 rtt := prometheus.NewGaugeVec(
60 prometheus.GaugeOpts{
61 Name: "rtt",
62 Help: "Round-trip time in millisecond",
63 },
64 []string{"connection_id"},
65 )
66 prometheus.MustRegister(rtt)
67
68 rttMin := prometheus.NewGaugeVec(
69 prometheus.GaugeOpts{
70 Name: "rtt_min",
71 Help: "Shortest round-trip time in millisecond",
72 },
73 []string{"connection_id"},
74 )
75 prometheus.MustRegister(rttMin)
76
77 rttMax := prometheus.NewGaugeVec(
78 prometheus.GaugeOpts{
79 Name: "rtt_max",
80 Help: "Longest round-trip time in millisecond",
81 },
82 []string{"connection_id"},
83 )
84 prometheus.MustRegister(rttMax)
85
86 receiveWindowAve := prometheus.NewGaugeVec(
87 prometheus.GaugeOpts{
88 Name: "receive_window_ave",
89 Help: "Average receive window size in bytes",
90 },
91 []string{"connection_id"},
92 )
93 prometheus.MustRegister(receiveWindowAve)
94
95 sendWindowAve := prometheus.NewGaugeVec(
96 prometheus.GaugeOpts{
97 Name: "send_window_ave",
98 Help: "Average send window size in bytes",
99 },
100 []string{"connection_id"},
101 )
102 prometheus.MustRegister(sendWindowAve)
103
104 receiveWindowMin := prometheus.NewGaugeVec(
105 prometheus.GaugeOpts{
106 Name: "receive_window_min",
107 Help: "Smallest receive window size in bytes",
108 },
109 []string{"connection_id"},
110 )
111 prometheus.MustRegister(receiveWindowMin)
112
113 receiveWindowMax := prometheus.NewGaugeVec(
114 prometheus.GaugeOpts{
115 Name: "receive_window_max",
116 Help: "Largest receive window size in bytes",
117 },
118 []string{"connection_id"},
119 )
120 prometheus.MustRegister(receiveWindowMax)
121
122 sendWindowMin := prometheus.NewGaugeVec(
123 prometheus.GaugeOpts{
124 Name: "send_window_min",
125 Help: "Smallest send window size in bytes",
126 },
127 []string{"connection_id"},
128 )
129 prometheus.MustRegister(sendWindowMin)
130
131 sendWindowMax := prometheus.NewGaugeVec(
132 prometheus.GaugeOpts{
133 Name: "send_window_max",
134 Help: "Largest send window size in bytes",
135 },
136 []string{"connection_id"},
137 )
138 prometheus.MustRegister(sendWindowMax)
139
140 inBoundRateCurr := prometheus.NewGaugeVec(
141 prometheus.GaugeOpts{
142 Name: "inbound_bytes_per_sec_curr",
143 Help: "Current inbounding bytes per second, 0 if there is no incoming connection",
144 },
145 []string{"connection_id"},
146 )
147 prometheus.MustRegister(inBoundRateCurr)
148
149 inBoundRateMin := prometheus.NewGaugeVec(
150 prometheus.GaugeOpts{
151 Name: "inbound_bytes_per_sec_min",
152 Help: "Minimum non-zero inbounding bytes per second",
153 },
154 []string{"connection_id"},
155 )
156 prometheus.MustRegister(inBoundRateMin)
157
158 inBoundRateMax := prometheus.NewGaugeVec(
159 prometheus.GaugeOpts{
160 Name: "inbound_bytes_per_sec_max",
161 Help: "Maximum inbounding bytes per second",
162 },
163 []string{"connection_id"},
164 )
165 prometheus.MustRegister(inBoundRateMax)
166
167 outBoundRateCurr := prometheus.NewGaugeVec(
168 prometheus.GaugeOpts{
169 Name: "outbound_bytes_per_sec_curr",
170 Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic",
171 },
172 []string{"connection_id"},
173 )
174 prometheus.MustRegister(outBoundRateCurr)
175
176 outBoundRateMin := prometheus.NewGaugeVec(
177 prometheus.GaugeOpts{
178 Name: "outbound_bytes_per_sec_min",
179 Help: "Minimum non-zero outbounding bytes per second",
180 },
181 []string{"connection_id"},
182 )
183 prometheus.MustRegister(outBoundRateMin)
184
185 outBoundRateMax := prometheus.NewGaugeVec(
186 prometheus.GaugeOpts{
187 Name: "outbound_bytes_per_sec_max",
188 Help: "Maximum outbounding bytes per second",
189 },
190 []string{"connection_id"},
191 )
192 prometheus.MustRegister(outBoundRateMax)
193
194 compBytesBefore := prometheus.NewGaugeVec(
195 prometheus.GaugeOpts{
196 Name: "comp_bytes_before",
197 Help: "Bytes sent via cross-stream compression, pre compression",
198 },
199 []string{"connection_id"},
200 )
201 prometheus.MustRegister(compBytesBefore)
202
203 compBytesAfter := prometheus.NewGaugeVec(
204 prometheus.GaugeOpts{
205 Name: "comp_bytes_after",
206 Help: "Bytes sent via cross-stream compression, post compression",
207 },
208 []string{"connection_id"},
209 )
210 prometheus.MustRegister(compBytesAfter)
211
212 compRateAve := prometheus.NewGaugeVec(
213 prometheus.GaugeOpts{
214 Name: "comp_rate_ave",
215 Help: "Average outbound cross-stream compression ratio",
216 },
217 []string{"connection_id"},
218 )
219 prometheus.MustRegister(compRateAve)
220
221 return &muxerMetrics{
222 rtt: rtt,
223 rttMin: rttMin,
224 rttMax: rttMax,
225 receiveWindowAve: receiveWindowAve,
226 sendWindowAve: sendWindowAve,
227 receiveWindowMin: receiveWindowMin,
228 receiveWindowMax: receiveWindowMax,
229 sendWindowMin: sendWindowMin,
230 sendWindowMax: sendWindowMax,
231 inBoundRateCurr: inBoundRateCurr,
232 inBoundRateMin: inBoundRateMin,
233 inBoundRateMax: inBoundRateMax,
234 outBoundRateCurr: outBoundRateCurr,
235 outBoundRateMin: outBoundRateMin,
236 outBoundRateMax: outBoundRateMax,
237 compBytesBefore: compBytesBefore,
238 compBytesAfter: compBytesAfter,
239 compRateAve: compRateAve,
240 }
241}
242
243func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
244 m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
245 m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
246 m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
247 m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
248 m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
249 m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
250 m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
251 m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
252 m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
253 m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
254 m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
255 m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
256 m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
257 m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
258 m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
259 m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
260 m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
261 m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
262}
263
264func convertRTTMilliSec(t time.Duration) float64 {
265 return float64(t / time.Millisecond)
266}
267
268// Metrics that can be collected without asking the edge
269func NewTunnelMetrics() *TunnelMetrics {
270 haConnections := prometheus.NewGauge(
271 prometheus.GaugeOpts{
272 Name: "ha_connections",
273 Help: "Number of active ha connections",
274 })
275 prometheus.MustRegister(haConnections)
276
277 totalRequests := prometheus.NewCounter(
278 prometheus.CounterOpts{
279 Name: "total_requests",
280 Help: "Amount of requests proxied through all the tunnels",
281 })
282 prometheus.MustRegister(totalRequests)
283
284 requestsPerTunnel := prometheus.NewCounterVec(
285 prometheus.CounterOpts{
286 Name: "requests_per_tunnel",
287 Help: "Amount of requests proxied through each tunnel",
288 },
289 []string{"connection_id"},
290 )
291 prometheus.MustRegister(requestsPerTunnel)
292
293 concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
294 prometheus.GaugeOpts{
295 Name: "concurrent_requests_per_tunnel",
296 Help: "Concurrent requests proxied through each tunnel",
297 },
298 []string{"connection_id"},
299 )
300 prometheus.MustRegister(concurrentRequestsPerTunnel)
301
302 maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
303 prometheus.GaugeOpts{
304 Name: "max_concurrent_requests_per_tunnel",
305 Help: "Largest number of concurrent requests proxied through each tunnel so far",
306 },
307 []string{"connection_id"},
308 )
309 prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
310
311 timerRetries := prometheus.NewGauge(
312 prometheus.GaugeOpts{
313 Name: "timer_retries",
314 Help: "Unacknowledged heart beats count",
315 })
316 prometheus.MustRegister(timerRetries)
317
318 responseByCode := prometheus.NewCounterVec(
319 prometheus.CounterOpts{
320 Name: "response_by_code",
321 Help: "Count of responses by HTTP status code",
322 },
323 []string{"status_code"},
324 )
325 prometheus.MustRegister(responseByCode)
326
327 responseCodePerTunnel := prometheus.NewCounterVec(
328 prometheus.CounterOpts{
329 Name: "response_code_per_tunnel",
330 Help: "Count of responses by HTTP status code fore each tunnel",
331 },
332 []string{"connection_id", "status_code"},
333 )
334 prometheus.MustRegister(responseCodePerTunnel)
335
336 serverLocations := prometheus.NewGaugeVec(
337 prometheus.GaugeOpts{
338 Name: "server_locations",
339 Help: "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
340 },
341 []string{"connection_id", "location"},
342 )
343 prometheus.MustRegister(serverLocations)
344
345 return &TunnelMetrics{
346 haConnections: haConnections,
347 totalRequests: totalRequests,
348 requestsPerTunnel: requestsPerTunnel,
349 concurrentRequestsPerTunnel: concurrentRequestsPerTunnel,
350 concurrentRequests: make(map[string]uint64),
351 maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
352 maxConcurrentRequests: make(map[string]uint64),
353 timerRetries: timerRetries,
354 responseByCode: responseByCode,
355 responseCodePerTunnel: responseCodePerTunnel,
356 serverLocations: serverLocations,
357 oldServerLocations: make(map[string]string),
358 muxerMetrics: newMuxerMetrics(),
359 tunnelsHA: NewTunnelsForHA(),
360 }
361}
362
363func (t *TunnelMetrics) incrementHaConnections() {
364 t.haConnections.Inc()
365}
366
367func (t *TunnelMetrics) decrementHaConnections() {
368 t.haConnections.Dec()
369}
370
371func (t *TunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
372 t.muxerMetrics.update(connectionID, metrics)
373}
374
375func (t *TunnelMetrics) incrementRequests(connectionID string) {
376 t.concurrentRequestsLock.Lock()
377 var concurrentRequests uint64
378 var ok bool
379 if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
380 t.concurrentRequests[connectionID]++
381 concurrentRequests++
382 } else {
383 t.concurrentRequests[connectionID] = 1
384 concurrentRequests = 1
385 }
386 if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
387 t.maxConcurrentRequests[connectionID] = concurrentRequests
388 t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
389 }
390 t.concurrentRequestsLock.Unlock()
391
392 t.totalRequests.Inc()
393 t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
394 t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
395}
396
397func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
398 t.concurrentRequestsLock.Lock()
399 if _, ok := t.concurrentRequests[connectionID]; ok {
400 t.concurrentRequests[connectionID]--
401 }
402 t.concurrentRequestsLock.Unlock()
403
404 t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
405}
406
407func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
408 t.responseByCode.WithLabelValues(code).Inc()
409 t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
410
411}
412
413func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
414 t.locationLock.Lock()
415 defer t.locationLock.Unlock()
416 if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
417 return
418 } else if ok {
419 t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
420 }
421 t.serverLocations.WithLabelValues(connectionID, loc).Inc()
422 t.oldServerLocations[connectionID] = loc
423}
424