Skip to main content

risingwave_common_metrics/monitor/
connection.rs

1// Copyright 2024 RisingWave Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::type_name;
16use std::cmp::Ordering;
17use std::future::Future;
18use std::io::{Error, IoSlice};
19use std::net::SocketAddr;
20use std::pin::Pin;
21use std::sync::LazyLock;
22use std::task::{Context, Poll};
23use std::time::Duration;
24
25use cfg_or_panic::cfg_or_panic;
26use futures::FutureExt;
27use http::Uri;
28use hyper_util::client::legacy::connect::dns::{GaiAddrs, GaiFuture, GaiResolver, Name};
29use hyper_util::client::legacy::connect::{Connected, Connection, HttpConnector};
30use hyper_util::rt::TokioIo;
31use itertools::Itertools;
32use pin_project_lite::pin_project;
33use prometheus::{
34    IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
35    register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry,
36};
37use thiserror_ext::AsReport;
38use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
39use tonic::transport::{Channel, Endpoint};
40use tower_service::Service;
41use tracing::{trace, warn};
42
43use crate::monitor::GLOBAL_METRICS_REGISTRY;
44use crate::{LabelGuardedIntCounterVec, register_guarded_int_counter_vec_with_registry};
45
46#[auto_impl::auto_impl(&mut)]
47pub trait MonitorAsyncReadWrite {
48    fn on_read(&mut self, _size: usize) {}
49    fn on_eof(&mut self) {}
50    fn on_read_err(&mut self, _err: &std::io::Error) {}
51
52    fn on_write(&mut self, _size: usize) {}
53    fn on_flush(&mut self) {}
54    fn on_shutdown(&mut self) {}
55    fn on_write_err(&mut self, _err: &std::io::Error) {}
56}
57
58pin_project! {
59    #[derive(Clone)]
60    pub struct MonitoredConnection<C, M> {
61        #[pin]
62        inner: C,
63        monitor: M,
64    }
65}
66
67impl<C, M> MonitoredConnection<C, M> {
68    pub fn new(connector: C, monitor: M) -> Self {
69        Self {
70            inner: connector,
71            monitor,
72        }
73    }
74
75    fn project_into(this: Pin<&mut Self>) -> (Pin<&mut C>, &mut M) {
76        let this = this.project();
77        (this.inner, this.monitor)
78    }
79
80    /// Delegate async read/write traits between tokio and hyper.
81    fn hyper_tokio_delegate(
82        self: Pin<&mut Self>,
83    ) -> TokioIo<MonitoredConnection<TokioIo<Pin<&mut C>>, &mut M>> {
84        let (inner, monitor) = MonitoredConnection::project_into(self);
85        TokioIo::new(MonitoredConnection::new(TokioIo::new(inner), monitor))
86    }
87}
88
89impl<C: AsyncRead, M: MonitorAsyncReadWrite> AsyncRead for MonitoredConnection<C, M> {
90    fn poll_read(
91        self: Pin<&mut Self>,
92        cx: &mut Context<'_>,
93        buf: &mut ReadBuf<'_>,
94    ) -> Poll<std::io::Result<()>> {
95        let before_buf_size = buf.filled().len();
96        let (inner, monitor) = MonitoredConnection::project_into(self);
97        let ret = inner.poll_read(cx, buf);
98        match &ret {
99            Poll::Ready(Ok(())) => {
100                let after_buf_size = buf.filled().len();
101                match after_buf_size.cmp(&before_buf_size) {
102                    Ordering::Less => {
103                        unreachable!(
104                            "buf size decrease after poll read. Bad AsyncRead implementation on {}",
105                            type_name::<C>()
106                        );
107                    }
108                    Ordering::Equal => {
109                        monitor.on_eof();
110                    }
111                    Ordering::Greater => {
112                        monitor.on_read(after_buf_size - before_buf_size);
113                    }
114                }
115            }
116            Poll::Ready(Err(e)) => {
117                monitor.on_read_err(e);
118            }
119            Poll::Pending => {}
120        }
121        ret
122    }
123}
124
125impl<C: hyper::rt::Read, M: MonitorAsyncReadWrite> hyper::rt::Read for MonitoredConnection<C, M> {
126    fn poll_read(
127        self: Pin<&mut Self>,
128        cx: &mut Context<'_>,
129        buf: hyper::rt::ReadBufCursor<'_>,
130    ) -> Poll<Result<(), std::io::Error>> {
131        hyper::rt::Read::poll_read(std::pin::pin!(self.hyper_tokio_delegate()), cx, buf)
132    }
133}
134
135impl<C: AsyncWrite, M: MonitorAsyncReadWrite> AsyncWrite for MonitoredConnection<C, M> {
136    fn poll_write(
137        self: Pin<&mut Self>,
138        cx: &mut Context<'_>,
139        buf: &[u8],
140    ) -> Poll<Result<usize, Error>> {
141        let (inner, monitor) = MonitoredConnection::project_into(self);
142        let ret = inner.poll_write(cx, buf);
143        match &ret {
144            Poll::Ready(Ok(size)) => {
145                monitor.on_write(*size);
146            }
147            Poll::Ready(Err(e)) => {
148                monitor.on_write_err(e);
149            }
150            Poll::Pending => {}
151        }
152        ret
153    }
154
155    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
156        let (inner, monitor) = MonitoredConnection::project_into(self);
157        let ret = inner.poll_flush(cx);
158        match &ret {
159            Poll::Ready(Ok(())) => {
160                monitor.on_flush();
161            }
162            Poll::Ready(Err(e)) => {
163                monitor.on_write_err(e);
164            }
165            Poll::Pending => {}
166        }
167        ret
168    }
169
170    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
171        let (inner, monitor) = MonitoredConnection::project_into(self);
172        let ret = inner.poll_shutdown(cx);
173        match &ret {
174            Poll::Ready(result) => {
175                monitor.on_shutdown();
176                if let Err(e) = result {
177                    monitor.on_write_err(e);
178                }
179            }
180            Poll::Pending => {}
181        }
182        ret
183    }
184
185    fn poll_write_vectored(
186        self: Pin<&mut Self>,
187        cx: &mut Context<'_>,
188        bufs: &[IoSlice<'_>],
189    ) -> Poll<Result<usize, Error>> {
190        let (inner, monitor) = MonitoredConnection::project_into(self);
191        let ret = inner.poll_write_vectored(cx, bufs);
192        match &ret {
193            Poll::Ready(Ok(size)) => {
194                monitor.on_write(*size);
195            }
196            Poll::Ready(Err(e)) => {
197                monitor.on_write_err(e);
198            }
199            Poll::Pending => {}
200        }
201        ret
202    }
203
204    fn is_write_vectored(&self) -> bool {
205        self.inner.is_write_vectored()
206    }
207}
208
209impl<C: hyper::rt::Write, M: MonitorAsyncReadWrite> hyper::rt::Write for MonitoredConnection<C, M> {
210    fn poll_write(
211        self: Pin<&mut Self>,
212        cx: &mut Context<'_>,
213        buf: &[u8],
214    ) -> Poll<Result<usize, std::io::Error>> {
215        hyper::rt::Write::poll_write(std::pin::pin!(self.hyper_tokio_delegate()), cx, buf)
216    }
217
218    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
219        hyper::rt::Write::poll_flush(std::pin::pin!(self.hyper_tokio_delegate()), cx)
220    }
221
222    fn poll_shutdown(
223        self: Pin<&mut Self>,
224        cx: &mut Context<'_>,
225    ) -> Poll<Result<(), std::io::Error>> {
226        hyper::rt::Write::poll_shutdown(std::pin::pin!(self.hyper_tokio_delegate()), cx)
227    }
228
229    fn is_write_vectored(&self) -> bool {
230        self.inner.is_write_vectored()
231    }
232
233    fn poll_write_vectored(
234        self: Pin<&mut Self>,
235        cx: &mut Context<'_>,
236        bufs: &[std::io::IoSlice<'_>],
237    ) -> Poll<Result<usize, std::io::Error>> {
238        hyper::rt::Write::poll_write_vectored(std::pin::pin!(self.hyper_tokio_delegate()), cx, bufs)
239    }
240}
241
242impl<C: Connection, M> Connection for MonitoredConnection<C, M> {
243    fn connected(&self) -> Connected {
244        self.inner.connected()
245    }
246}
247
248#[cfg(not(madsim))]
249impl<C: tonic::transport::server::Connected, M> tonic::transport::server::Connected
250    for MonitoredConnection<C, M>
251{
252    type ConnectInfo = C::ConnectInfo;
253
254    fn connect_info(&self) -> Self::ConnectInfo {
255        self.inner.connect_info()
256    }
257}
258
259pub trait MonitorNewConnection {
260    type ConnectionMonitor: MonitorAsyncReadWrite;
261
262    fn new_connection_monitor(&self, endpoint: String) -> Self::ConnectionMonitor;
263    fn on_err(&self, endpoint: String);
264}
265
266impl<C: Service<Uri>, M: MonitorNewConnection + Clone + 'static> Service<Uri>
267    for MonitoredConnection<C, M>
268where
269    C::Future: 'static,
270{
271    type Error = C::Error;
272    type Response = MonitoredConnection<C::Response, M::ConnectionMonitor>;
273
274    type Future = impl Future<Output = Result<Self::Response, Self::Error>> + 'static;
275
276    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
277        let ret = self.inner.poll_ready(cx);
278        if let Poll::Ready(Err(_)) = &ret {
279            self.monitor.on_err("<poll_ready>".to_owned());
280        }
281        ret
282    }
283
284    fn call(&mut self, uri: Uri) -> Self::Future {
285        let endpoint = format!("{:?}", uri.host());
286        let monitor = self.monitor.clone();
287        self.inner
288            .call(uri)
289            .map(move |result: Result<_, _>| match result {
290                Ok(resp) => Ok(MonitoredConnection::new(
291                    resp,
292                    monitor.new_connection_monitor(endpoint),
293                )),
294                Err(e) => {
295                    monitor.on_err(endpoint);
296                    Err(e)
297                }
298            })
299    }
300}
301
302#[cfg(not(madsim))]
303impl<Con, E, C: futures::Stream<Item = Result<Con, E>>, M: MonitorNewConnection> futures::Stream
304    for MonitoredConnection<C, M>
305where
306    Con:
307        tonic::transport::server::Connected<ConnectInfo = tonic::transport::server::TcpConnectInfo>,
308{
309    type Item = Result<MonitoredConnection<Con, M::ConnectionMonitor>, E>;
310
311    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
312        let (inner, monitor) = MonitoredConnection::project_into(self);
313        inner.poll_next(cx).map(|opt| {
314            opt.map(|result| {
315                result.map(|conn| {
316                    let remote_addr = conn.connect_info().remote_addr();
317                    let endpoint = remote_addr
318                        .map(|remote_addr| format!("{}", remote_addr.ip()))
319                        .unwrap_or("unknown".to_owned());
320                    MonitoredConnection::new(conn, monitor.new_connection_monitor(endpoint))
321                })
322            })
323        })
324    }
325
326    fn size_hint(&self) -> (usize, Option<usize>) {
327        self.inner.size_hint()
328    }
329}
330
331#[derive(Clone)]
332pub struct ConnectionMetrics {
333    connection_count: IntGaugeVec,
334    connection_create_rate: IntCounterVec,
335    connection_err_rate: IntCounterVec,
336
337    read_rate: IntCounterVec,
338    reader_count: IntGaugeVec,
339
340    write_rate: IntCounterVec,
341    writer_count: IntGaugeVec,
342
343    io_err_rate: LabelGuardedIntCounterVec,
344}
345
346pub static GLOBAL_CONNECTION_METRICS: LazyLock<ConnectionMetrics> =
347    LazyLock::new(|| ConnectionMetrics::new(&GLOBAL_METRICS_REGISTRY));
348
349impl ConnectionMetrics {
350    pub fn new(registry: &Registry) -> Self {
351        let labels = ["connection_type", "uri"];
352        let connection_count = register_int_gauge_vec_with_registry!(
353            "connection_count",
354            "The number of current existing connection",
355            &labels,
356            registry,
357        )
358        .unwrap();
359
360        let connection_create_rate = register_int_counter_vec_with_registry!(
361            "connection_create_rate",
362            "Rate on creating new connection",
363            &labels,
364            registry,
365        )
366        .unwrap();
367
368        let connection_err_rate = register_int_counter_vec_with_registry!(
369            "connection_err_rate",
370            "Error rate on creating new connection",
371            &labels,
372            registry,
373        )
374        .unwrap();
375
376        let read_rate = register_int_counter_vec_with_registry!(
377            "connection_read_rate",
378            "Read rate of a connection",
379            &labels,
380            registry,
381        )
382        .unwrap();
383
384        let reader_count = register_int_gauge_vec_with_registry!(
385            "connection_reader_count",
386            "The number of current existing reader",
387            &labels,
388            registry,
389        )
390        .unwrap();
391
392        let write_rate = register_int_counter_vec_with_registry!(
393            "connection_write_rate",
394            "Write rate of a connection",
395            &labels,
396            registry,
397        )
398        .unwrap();
399
400        let writer_count = register_int_gauge_vec_with_registry!(
401            "connection_writer_count",
402            "The number of current existing writer",
403            &labels,
404            registry,
405        )
406        .unwrap();
407
408        let io_err_rate = register_guarded_int_counter_vec_with_registry!(
409            "connection_io_err_rate",
410            "IO err rate of a connection",
411            &["connection_type", "uri", "op_type", "error_kind"],
412            registry,
413        )
414        .unwrap();
415
416        Self {
417            connection_count,
418            connection_create_rate,
419            connection_err_rate,
420            read_rate,
421            reader_count,
422            write_rate,
423            writer_count,
424            io_err_rate,
425        }
426    }
427}
428
429pub struct TcpConfig {
430    pub tcp_nodelay: bool,
431    pub keepalive_duration: Option<Duration>,
432}
433
434impl Default for TcpConfig {
435    fn default() -> Self {
436        Self {
437            // Without it, Nagle's algorithm holds the DATA frame of a small gRPC request until the
438            // peer's delayed ACK, adding about 40ms to the request.
439            tcp_nodelay: true,
440            keepalive_duration: None,
441        }
442    }
443}
444
445pub fn monitor_connector<C>(
446    connector: C,
447    connection_type: impl Into<String>,
448) -> MonitoredConnection<C, MonitorNewConnectionImpl> {
449    let connection_type = connection_type.into();
450    trace!(
451        "monitoring connector {} with type {}",
452        type_name::<C>(),
453        connection_type
454    );
455    MonitoredConnection::new(connector, MonitorNewConnectionImpl { connection_type })
456}
457
458pub struct MonitoredGaiAddrs {
459    inner: Vec<SocketAddr>,
460    pos: usize,
461}
462
463impl From<GaiAddrs> for MonitoredGaiAddrs {
464    fn from(value: GaiAddrs) -> Self {
465        Self {
466            inner: value.collect_vec(),
467            pos: 0,
468        }
469    }
470}
471
472impl Iterator for MonitoredGaiAddrs {
473    type Item = SocketAddr;
474
475    fn next(&mut self) -> Option<Self::Item> {
476        let res = self.inner.get(self.pos).cloned();
477        self.pos += 1;
478        res
479    }
480}
481
482pub struct MonitoredGaiFuture {
483    name: Name,
484    inner: GaiFuture,
485}
486
487impl std::fmt::Debug for MonitoredGaiFuture {
488    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
489        f.pad("MonitoredGaiFuture")
490    }
491}
492
493impl Future for MonitoredGaiFuture {
494    type Output = Result<MonitoredGaiAddrs, Error>;
495
496    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
497        Pin::new(&mut self.inner).poll(cx).map(|res| match res {
498            Ok(addrs) => {
499                let addrs: MonitoredGaiAddrs = addrs.into();
500                trace!("resolve {} => {:?}", self.name, addrs.inner);
501                Ok(addrs)
502            }
503            Err(err) => Err(err),
504        })
505    }
506}
507
508#[derive(Clone)]
509pub struct MonitoredGaiResolver {
510    inner: GaiResolver,
511}
512
513impl std::fmt::Debug for MonitoredGaiResolver {
514    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
515        f.pad("MonitoredGaiResolver")
516    }
517}
518
519impl Default for MonitoredGaiResolver {
520    fn default() -> Self {
521        Self {
522            inner: GaiResolver::new(),
523        }
524    }
525}
526
527impl Service<Name> for MonitoredGaiResolver {
528    type Error = Error;
529    type Future = MonitoredGaiFuture;
530    type Response = MonitoredGaiAddrs;
531
532    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
533        self.inner.poll_ready(cx)
534    }
535
536    fn call(&mut self, req: Name) -> Self::Future {
537        MonitoredGaiFuture {
538            name: req.clone(),
539            inner: self.inner.call(req),
540        }
541    }
542}
543
544#[cfg_or_panic(not(madsim))]
545fn monitored_http_connector(
546    connection_type: impl Into<String>,
547    config: TcpConfig,
548) -> MonitoredConnection<HttpConnector<MonitoredGaiResolver>, MonitorNewConnectionImpl> {
549    let resolver = MonitoredGaiResolver::default();
550    let mut http = HttpConnector::new_with_resolver(resolver);
551
552    http.enforce_http(false);
553    http.set_nodelay(config.tcp_nodelay);
554    http.set_keepalive(config.keepalive_duration);
555
556    monitor_connector(http, connection_type)
557}
558
559/// Attach general configurations to the endpoint.
560#[cfg_or_panic(not(madsim))]
561fn configure_endpoint(endpoint: Endpoint) -> Endpoint {
562    // This is to mitigate https://github.com/risingwavelabs/risingwave/issues/18039.
563    // TODO: remove this after https://github.com/hyperium/hyper/issues/3724 gets resolved.
564    endpoint.http2_max_header_list_size(16 * 1024 * 1024)
565}
566
567#[easy_ext::ext(EndpointExt)]
568impl Endpoint {
569    pub async fn monitored_connect(
570        mut self,
571        connection_type: impl Into<String>,
572        config: TcpConfig,
573    ) -> Result<Channel, tonic::transport::Error> {
574        #[cfg(not(madsim))]
575        {
576            self = configure_endpoint(self);
577            let connector = monitored_http_connector(connection_type, config);
578            self.connect_with_connector(connector).await
579        }
580        #[cfg(madsim)]
581        {
582            self.connect().await
583        }
584    }
585
586    #[cfg(not(madsim))]
587    pub fn monitored_connect_lazy(
588        mut self,
589        connection_type: impl Into<String>,
590        config: TcpConfig,
591    ) -> Channel {
592        self = configure_endpoint(self);
593        let connector = monitored_http_connector(connection_type, config);
594        self.connect_with_connector_lazy(connector)
595    }
596}
597
598#[cfg(not(madsim))]
599#[easy_ext::ext(RouterExt)]
600impl<L> tonic::transport::server::Router<L> {
601    /// Serve the given service while monitoring the connection.
602    ///
603    /// Calling the function will first bind the given service to the given address. Awaiting the
604    /// returned future will then start the server and keep it running until the given signal
605    /// future resolves.
606    pub fn monitored_serve_with_shutdown<ResBody>(
607        self,
608        listen_addr: std::net::SocketAddr,
609        connection_type: impl Into<String>,
610        config: TcpConfig,
611        signal: impl Future<Output = ()>,
612    ) -> impl Future<Output = ()>
613    where
614        L: tower_layer::Layer<tonic::service::Routes>,
615        L::Service: Service<http::Request<tonic::body::Body>, Response = http::Response<ResBody>>
616            + Clone
617            + Send
618            + 'static,
619        <<L as tower_layer::Layer<tonic::service::Routes>>::Service as Service<
620            http::Request<tonic::body::Body>,
621        >>::Future: Send + 'static,
622        <<L as tower_layer::Layer<tonic::service::Routes>>::Service as Service<
623            http::Request<tonic::body::Body>,
624        >>::Error: Into<Box<dyn std::error::Error + Send + Sync>> + Send,
625        ResBody: http_body::Body<Data = bytes::Bytes> + Send + 'static,
626        ResBody::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
627    {
628        let connection_type = connection_type.into();
629        let incoming = tonic::transport::server::TcpIncoming::bind(listen_addr)
630            .map(|incoming| {
631                incoming
632                    .with_nodelay(Some(config.tcp_nodelay))
633                    .with_keepalive(config.keepalive_duration)
634            })
635            .unwrap_or_else(|err| {
636                panic!(
637                    "failed to bind `{connection_type}` to `{listen_addr}`: {}",
638                    err.as_report()
639                )
640            });
641        let incoming =
642            MonitoredConnection::new(incoming, MonitorNewConnectionImpl { connection_type });
643
644        async move {
645            self.serve_with_incoming_shutdown(incoming, signal)
646                .await
647                .unwrap()
648        }
649    }
650}
651
652#[cfg(madsim)]
653#[easy_ext::ext(RouterExt)]
654impl<L> tonic::transport::server::Router<L> {
655    pub async fn monitored_serve_with_shutdown(
656        self,
657        listen_addr: std::net::SocketAddr,
658        connection_type: impl Into<String>,
659        config: TcpConfig,
660        signal: impl Future<Output = ()>,
661    ) {
662        self.serve_with_shutdown(listen_addr, signal).await.unwrap()
663    }
664}
665
666#[derive(Clone)]
667pub struct MonitorNewConnectionImpl {
668    connection_type: String,
669}
670
671impl MonitorNewConnection for MonitorNewConnectionImpl {
672    type ConnectionMonitor = MonitorAsyncReadWriteImpl;
673
674    fn new_connection_monitor(&self, endpoint: String) -> Self::ConnectionMonitor {
675        let labels = [self.connection_type.as_str(), endpoint.as_str()];
676        let read_rate = GLOBAL_CONNECTION_METRICS
677            .read_rate
678            .with_label_values(&labels);
679        let reader_count = GLOBAL_CONNECTION_METRICS
680            .reader_count
681            .with_label_values(&labels);
682        let write_rate = GLOBAL_CONNECTION_METRICS
683            .write_rate
684            .with_label_values(&labels);
685        let writer_count = GLOBAL_CONNECTION_METRICS
686            .writer_count
687            .with_label_values(&labels);
688        let connection_count = GLOBAL_CONNECTION_METRICS
689            .connection_count
690            .with_label_values(&labels);
691
692        GLOBAL_CONNECTION_METRICS
693            .connection_create_rate
694            .with_label_values(&labels)
695            .inc();
696
697        MonitorAsyncReadWriteImpl::new(
698            endpoint,
699            self.connection_type.clone(),
700            read_rate,
701            reader_count,
702            write_rate,
703            writer_count,
704            connection_count,
705        )
706    }
707
708    fn on_err(&self, endpoint: String) {
709        GLOBAL_CONNECTION_METRICS
710            .connection_err_rate
711            .with_label_values(&[self.connection_type.as_str(), endpoint.as_str()])
712            .inc();
713    }
714}
715
716const READ_WRITE_RATE_REPORT_INTERVAL: u64 = 1024;
717
718pub struct MonitorAsyncReadWriteImpl {
719    endpoint: String,
720    connection_type: String,
721
722    unreported_read_rate: u64,
723    read_rate: IntCounter,
724    reader_count_guard: IntGauge,
725    is_eof: bool,
726
727    unreported_write_rate: u64,
728    write_rate: IntCounter,
729    writer_count_guard: IntGauge,
730    is_shutdown: bool,
731
732    connection_count_guard: IntGauge,
733}
734
735impl MonitorAsyncReadWriteImpl {
736    pub fn new(
737        endpoint: String,
738        connection_type: String,
739        read_rate: IntCounter,
740        reader_count: IntGauge,
741        write_rate: IntCounter,
742        writer_count: IntGauge,
743        connection_count: IntGauge,
744    ) -> Self {
745        reader_count.inc();
746        writer_count.inc();
747        connection_count.inc();
748        Self {
749            endpoint,
750            connection_type,
751            unreported_read_rate: 0,
752            read_rate,
753            reader_count_guard: reader_count,
754            is_eof: false,
755            unreported_write_rate: 0,
756            write_rate,
757            writer_count_guard: writer_count,
758            is_shutdown: false,
759            connection_count_guard: connection_count,
760        }
761    }
762}
763
764impl Drop for MonitorAsyncReadWriteImpl {
765    fn drop(&mut self) {
766        if self.unreported_read_rate > 0 {
767            self.read_rate.inc_by(self.unreported_read_rate);
768        }
769        if self.unreported_write_rate > 0 {
770            self.write_rate.inc_by(self.unreported_write_rate);
771        }
772        if !self.is_eof {
773            self.reader_count_guard.dec();
774        }
775        if !self.is_shutdown {
776            self.writer_count_guard.dec();
777        }
778        self.connection_count_guard.dec();
779    }
780}
781
782impl MonitorAsyncReadWrite for MonitorAsyncReadWriteImpl {
783    fn on_read(&mut self, size: usize) {
784        self.unreported_read_rate += size as u64;
785        if self.unreported_read_rate >= READ_WRITE_RATE_REPORT_INTERVAL {
786            self.read_rate.inc_by(self.unreported_read_rate);
787            self.unreported_read_rate = 0;
788        }
789    }
790
791    fn on_eof(&mut self) {
792        if self.is_eof {
793            warn!("get eof for multiple time");
794            return;
795        }
796        self.is_eof = true;
797        self.reader_count_guard.dec();
798    }
799
800    #[allow(rw::temporary_guarded_metric)]
801    fn on_read_err(&mut self, err: &Error) {
802        // This is intentionally an ephemeral event counter. Collection exports
803        // the dropped label once before removing it.
804        GLOBAL_CONNECTION_METRICS
805            .io_err_rate
806            .with_guarded_label_values(&[
807                self.connection_type.as_str(),
808                self.endpoint.as_str(),
809                "read",
810                err.kind().to_string().as_str(),
811            ])
812            .inc();
813    }
814
815    fn on_write(&mut self, size: usize) {
816        self.unreported_write_rate += size as u64;
817        if self.unreported_write_rate >= READ_WRITE_RATE_REPORT_INTERVAL {
818            self.write_rate.inc_by(self.unreported_write_rate);
819            self.unreported_write_rate = 0;
820        }
821    }
822
823    fn on_shutdown(&mut self) {
824        if self.is_shutdown {
825            warn!("get shutdown for multiple time");
826            return;
827        }
828        self.is_shutdown = true;
829        self.writer_count_guard.dec();
830    }
831
832    #[allow(rw::temporary_guarded_metric)]
833    fn on_write_err(&mut self, err: &Error) {
834        // This is intentionally an ephemeral event counter. Collection exports
835        // the dropped label once before removing it.
836        GLOBAL_CONNECTION_METRICS
837            .io_err_rate
838            .with_guarded_label_values(&[
839                self.connection_type.as_str(),
840                self.endpoint.as_str(),
841                "write",
842                err.kind().to_string().as_str(),
843            ])
844            .inc();
845    }
846}