GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

cloudflare/cloudflared

Public

mirrored from https://github.com/cloudflare/cloudflaredAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
2018.10.1

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

cloudflared/origin

origin/metrics.go

421lines · modecode

RawDownload
Latest commit unavailable.
1package origin
2
3import (
4 "sync"
5 "time"
6
7 "github.com/cloudflare/cloudflared/h2mux"
8
9 "github.com/prometheus/client_golang/prometheus"
10)
11
12type muxerMetrics struct {
13 rtt *prometheus.GaugeVec
14 rttMin *prometheus.GaugeVec
15 rttMax *prometheus.GaugeVec
16 receiveWindowAve *prometheus.GaugeVec
17 sendWindowAve *prometheus.GaugeVec
18 receiveWindowMin *prometheus.GaugeVec
19 receiveWindowMax *prometheus.GaugeVec
20 sendWindowMin *prometheus.GaugeVec
21 sendWindowMax *prometheus.GaugeVec
22 inBoundRateCurr *prometheus.GaugeVec
23 inBoundRateMin *prometheus.GaugeVec
24 inBoundRateMax *prometheus.GaugeVec
25 outBoundRateCurr *prometheus.GaugeVec
26 outBoundRateMin *prometheus.GaugeVec
27 outBoundRateMax *prometheus.GaugeVec
28 compBytesBefore *prometheus.GaugeVec
29 compBytesAfter *prometheus.GaugeVec
30 compRateAve *prometheus.GaugeVec
31}
32
33type TunnelMetrics struct {
34 haConnections prometheus.Gauge
35 totalRequests prometheus.Counter
36 requestsPerTunnel *prometheus.CounterVec
37 // concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
38 concurrentRequestsLock sync.Mutex
39 concurrentRequestsPerTunnel *prometheus.GaugeVec
40 // concurrentRequests records count of concurrent requests for each tunnel
41 concurrentRequests map[string]uint64
42 maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
43 // concurrentRequests records max count of concurrent requests for each tunnel
44 maxConcurrentRequests map[string]uint64
45 timerRetries prometheus.Gauge
46 responseByCode *prometheus.CounterVec
47 responseCodePerTunnel *prometheus.CounterVec
48 serverLocations *prometheus.GaugeVec
49 // locationLock is a mutex for oldServerLocations
50 locationLock sync.Mutex
51 // oldServerLocations stores the last server the tunnel was connected to
52 oldServerLocations map[string]string
53
54 muxerMetrics *muxerMetrics
55}
56
57func newMuxerMetrics() *muxerMetrics {
58 rtt := prometheus.NewGaugeVec(
59 prometheus.GaugeOpts{
60 Name: "rtt",
61 Help: "Round-trip time in millisecond",
62 },
63 []string{"connection_id"},
64 )
65 prometheus.MustRegister(rtt)
66
67 rttMin := prometheus.NewGaugeVec(
68 prometheus.GaugeOpts{
69 Name: "rtt_min",
70 Help: "Shortest round-trip time in millisecond",
71 },
72 []string{"connection_id"},
73 )
74 prometheus.MustRegister(rttMin)
75
76 rttMax := prometheus.NewGaugeVec(
77 prometheus.GaugeOpts{
78 Name: "rtt_max",
79 Help: "Longest round-trip time in millisecond",
80 },
81 []string{"connection_id"},
82 )
83 prometheus.MustRegister(rttMax)
84
85 receiveWindowAve := prometheus.NewGaugeVec(
86 prometheus.GaugeOpts{
87 Name: "receive_window_ave",
88 Help: "Average receive window size in bytes",
89 },
90 []string{"connection_id"},
91 )
92 prometheus.MustRegister(receiveWindowAve)
93
94 sendWindowAve := prometheus.NewGaugeVec(
95 prometheus.GaugeOpts{
96 Name: "send_window_ave",
97 Help: "Average send window size in bytes",
98 },
99 []string{"connection_id"},
100 )
101 prometheus.MustRegister(sendWindowAve)
102
103 receiveWindowMin := prometheus.NewGaugeVec(
104 prometheus.GaugeOpts{
105 Name: "receive_window_min",
106 Help: "Smallest receive window size in bytes",
107 },
108 []string{"connection_id"},
109 )
110 prometheus.MustRegister(receiveWindowMin)
111
112 receiveWindowMax := prometheus.NewGaugeVec(
113 prometheus.GaugeOpts{
114 Name: "receive_window_max",
115 Help: "Largest receive window size in bytes",
116 },
117 []string{"connection_id"},
118 )
119 prometheus.MustRegister(receiveWindowMax)
120
121 sendWindowMin := prometheus.NewGaugeVec(
122 prometheus.GaugeOpts{
123 Name: "send_window_min",
124 Help: "Smallest send window size in bytes",
125 },
126 []string{"connection_id"},
127 )
128 prometheus.MustRegister(sendWindowMin)
129
130 sendWindowMax := prometheus.NewGaugeVec(
131 prometheus.GaugeOpts{
132 Name: "send_window_max",
133 Help: "Largest send window size in bytes",
134 },
135 []string{"connection_id"},
136 )
137 prometheus.MustRegister(sendWindowMax)
138
139 inBoundRateCurr := prometheus.NewGaugeVec(
140 prometheus.GaugeOpts{
141 Name: "inbound_bytes_per_sec_curr",
142 Help: "Current inbounding bytes per second, 0 if there is no incoming connection",
143 },
144 []string{"connection_id"},
145 )
146 prometheus.MustRegister(inBoundRateCurr)
147
148 inBoundRateMin := prometheus.NewGaugeVec(
149 prometheus.GaugeOpts{
150 Name: "inbound_bytes_per_sec_min",
151 Help: "Minimum non-zero inbounding bytes per second",
152 },
153 []string{"connection_id"},
154 )
155 prometheus.MustRegister(inBoundRateMin)
156
157 inBoundRateMax := prometheus.NewGaugeVec(
158 prometheus.GaugeOpts{
159 Name: "inbound_bytes_per_sec_max",
160 Help: "Maximum inbounding bytes per second",
161 },
162 []string{"connection_id"},
163 )
164 prometheus.MustRegister(inBoundRateMax)
165
166 outBoundRateCurr := prometheus.NewGaugeVec(
167 prometheus.GaugeOpts{
168 Name: "outbound_bytes_per_sec_curr",
169 Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic",
170 },
171 []string{"connection_id"},
172 )
173 prometheus.MustRegister(outBoundRateCurr)
174
175 outBoundRateMin := prometheus.NewGaugeVec(
176 prometheus.GaugeOpts{
177 Name: "outbound_bytes_per_sec_min",
178 Help: "Minimum non-zero outbounding bytes per second",
179 },
180 []string{"connection_id"},
181 )
182 prometheus.MustRegister(outBoundRateMin)
183
184 outBoundRateMax := prometheus.NewGaugeVec(
185 prometheus.GaugeOpts{
186 Name: "outbound_bytes_per_sec_max",
187 Help: "Maximum outbounding bytes per second",
188 },
189 []string{"connection_id"},
190 )
191 prometheus.MustRegister(outBoundRateMax)
192
193 compBytesBefore := prometheus.NewGaugeVec(
194 prometheus.GaugeOpts{
195 Name: "comp_bytes_before",
196 Help: "Bytes sent via cross-stream compression, pre compression",
197 },
198 []string{"connection_id"},
199 )
200 prometheus.MustRegister(compBytesBefore)
201
202 compBytesAfter := prometheus.NewGaugeVec(
203 prometheus.GaugeOpts{
204 Name: "comp_bytes_after",
205 Help: "Bytes sent via cross-stream compression, post compression",
206 },
207 []string{"connection_id"},
208 )
209 prometheus.MustRegister(compBytesAfter)
210
211 compRateAve := prometheus.NewGaugeVec(
212 prometheus.GaugeOpts{
213 Name: "comp_rate_ave",
214 Help: "Average outbound cross-stream compression ratio",
215 },
216 []string{"connection_id"},
217 )
218 prometheus.MustRegister(compRateAve)
219
220 return &muxerMetrics{
221 rtt: rtt,
222 rttMin: rttMin,
223 rttMax: rttMax,
224 receiveWindowAve: receiveWindowAve,
225 sendWindowAve: sendWindowAve,
226 receiveWindowMin: receiveWindowMin,
227 receiveWindowMax: receiveWindowMax,
228 sendWindowMin: sendWindowMin,
229 sendWindowMax: sendWindowMax,
230 inBoundRateCurr: inBoundRateCurr,
231 inBoundRateMin: inBoundRateMin,
232 inBoundRateMax: inBoundRateMax,
233 outBoundRateCurr: outBoundRateCurr,
234 outBoundRateMin: outBoundRateMin,
235 outBoundRateMax: outBoundRateMax,
236 compBytesBefore: compBytesBefore,
237 compBytesAfter: compBytesAfter,
238 compRateAve: compRateAve,
239 }
240}
241
242func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
243 m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
244 m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
245 m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
246 m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
247 m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
248 m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
249 m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
250 m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
251 m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
252 m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
253 m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
254 m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
255 m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
256 m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
257 m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
258 m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
259 m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
260 m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
261}
262
263func convertRTTMilliSec(t time.Duration) float64 {
264 return float64(t / time.Millisecond)
265}
266
267// Metrics that can be collected without asking the edge
268func NewTunnelMetrics() *TunnelMetrics {
269 haConnections := prometheus.NewGauge(
270 prometheus.GaugeOpts{
271 Name: "ha_connections",
272 Help: "Number of active ha connections",
273 })
274 prometheus.MustRegister(haConnections)
275
276 totalRequests := prometheus.NewCounter(
277 prometheus.CounterOpts{
278 Name: "total_requests",
279 Help: "Amount of requests proxied through all the tunnels",
280 })
281 prometheus.MustRegister(totalRequests)
282
283 requestsPerTunnel := prometheus.NewCounterVec(
284 prometheus.CounterOpts{
285 Name: "requests_per_tunnel",
286 Help: "Amount of requests proxied through each tunnel",
287 },
288 []string{"connection_id"},
289 )
290 prometheus.MustRegister(requestsPerTunnel)
291
292 concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
293 prometheus.GaugeOpts{
294 Name: "concurrent_requests_per_tunnel",
295 Help: "Concurrent requests proxied through each tunnel",
296 },
297 []string{"connection_id"},
298 )
299 prometheus.MustRegister(concurrentRequestsPerTunnel)
300
301 maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
302 prometheus.GaugeOpts{
303 Name: "max_concurrent_requests_per_tunnel",
304 Help: "Largest number of concurrent requests proxied through each tunnel so far",
305 },
306 []string{"connection_id"},
307 )
308 prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
309
310 timerRetries := prometheus.NewGauge(
311 prometheus.GaugeOpts{
312 Name: "timer_retries",
313 Help: "Unacknowledged heart beats count",
314 })
315 prometheus.MustRegister(timerRetries)
316
317 responseByCode := prometheus.NewCounterVec(
318 prometheus.CounterOpts{
319 Name: "response_by_code",
320 Help: "Count of responses by HTTP status code",
321 },
322 []string{"status_code"},
323 )
324 prometheus.MustRegister(responseByCode)
325
326 responseCodePerTunnel := prometheus.NewCounterVec(
327 prometheus.CounterOpts{
328 Name: "response_code_per_tunnel",
329 Help: "Count of responses by HTTP status code fore each tunnel",
330 },
331 []string{"connection_id", "status_code"},
332 )
333 prometheus.MustRegister(responseCodePerTunnel)
334
335 serverLocations := prometheus.NewGaugeVec(
336 prometheus.GaugeOpts{
337 Name: "server_locations",
338 Help: "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
339 },
340 []string{"connection_id", "location"},
341 )
342 prometheus.MustRegister(serverLocations)
343
344 return &TunnelMetrics{
345 haConnections: haConnections,
346 totalRequests: totalRequests,
347 requestsPerTunnel: requestsPerTunnel,
348 concurrentRequestsPerTunnel: concurrentRequestsPerTunnel,
349 concurrentRequests: make(map[string]uint64),
350 maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
351 maxConcurrentRequests: make(map[string]uint64),
352 timerRetries: timerRetries,
353 responseByCode: responseByCode,
354 responseCodePerTunnel: responseCodePerTunnel,
355 serverLocations: serverLocations,
356 oldServerLocations: make(map[string]string),
357 muxerMetrics: newMuxerMetrics(),
358 }
359}
360
361func (t *TunnelMetrics) incrementHaConnections() {
362 t.haConnections.Inc()
363}
364
365func (t *TunnelMetrics) decrementHaConnections() {
366 t.haConnections.Dec()
367}
368
369func (t *TunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
370 t.muxerMetrics.update(connectionID, metrics)
371}
372
373func (t *TunnelMetrics) incrementRequests(connectionID string) {
374 t.concurrentRequestsLock.Lock()
375 var concurrentRequests uint64
376 var ok bool
377 if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
378 t.concurrentRequests[connectionID] += 1
379 concurrentRequests++
380 } else {
381 t.concurrentRequests[connectionID] = 1
382 concurrentRequests = 1
383 }
384 if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
385 t.maxConcurrentRequests[connectionID] = concurrentRequests
386 t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
387 }
388 t.concurrentRequestsLock.Unlock()
389
390 t.totalRequests.Inc()
391 t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
392 t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
393}
394
395func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
396 t.concurrentRequestsLock.Lock()
397 if _, ok := t.concurrentRequests[connectionID]; ok {
398 t.concurrentRequests[connectionID] -= 1
399 }
400 t.concurrentRequestsLock.Unlock()
401
402 t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
403}
404
405func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
406 t.responseByCode.WithLabelValues(code).Inc()
407 t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
408
409}
410
411func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
412 t.locationLock.Lock()
413 defer t.locationLock.Unlock()
414 if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
415 return
416 } else if ok {
417 t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
418 }
419 t.serverLocations.WithLabelValues(connectionID, loc).Inc()
420 t.oldServerLocations[connectionID] = loc
421}
422