1use std::any::type_name;
16use std::cmp::Ordering;
17use std::future::Future;
18use std::io::{Error, IoSlice};
19use std::net::SocketAddr;
20use std::pin::Pin;
21use std::sync::LazyLock;
22use std::task::{Context, Poll};
23use std::time::Duration;
24
25use cfg_or_panic::cfg_or_panic;
26use futures::FutureExt;
27use http::Uri;
28use hyper_util::client::legacy::connect::dns::{GaiAddrs, GaiFuture, GaiResolver, Name};
29use hyper_util::client::legacy::connect::{Connected, Connection, HttpConnector};
30use hyper_util::rt::TokioIo;
31use itertools::Itertools;
32use pin_project_lite::pin_project;
33use prometheus::{
34 IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
35 register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry,
36};
37use thiserror_ext::AsReport;
38use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
39use tonic::transport::{Channel, Endpoint};
40use tower_service::Service;
41use tracing::{trace, warn};
42
43use crate::monitor::GLOBAL_METRICS_REGISTRY;
44use crate::{LabelGuardedIntCounterVec, register_guarded_int_counter_vec_with_registry};
45
46#[auto_impl::auto_impl(&mut)]
47pub trait MonitorAsyncReadWrite {
48 fn on_read(&mut self, _size: usize) {}
49 fn on_eof(&mut self) {}
50 fn on_read_err(&mut self, _err: &std::io::Error) {}
51
52 fn on_write(&mut self, _size: usize) {}
53 fn on_flush(&mut self) {}
54 fn on_shutdown(&mut self) {}
55 fn on_write_err(&mut self, _err: &std::io::Error) {}
56}
57
58pin_project! {
59 #[derive(Clone)]
60 pub struct MonitoredConnection<C, M> {
61 #[pin]
62 inner: C,
63 monitor: M,
64 }
65}
66
67impl<C, M> MonitoredConnection<C, M> {
68 pub fn new(connector: C, monitor: M) -> Self {
69 Self {
70 inner: connector,
71 monitor,
72 }
73 }
74
75 fn project_into(this: Pin<&mut Self>) -> (Pin<&mut C>, &mut M) {
76 let this = this.project();
77 (this.inner, this.monitor)
78 }
79
80 fn hyper_tokio_delegate(
82 self: Pin<&mut Self>,
83 ) -> TokioIo<MonitoredConnection<TokioIo<Pin<&mut C>>, &mut M>> {
84 let (inner, monitor) = MonitoredConnection::project_into(self);
85 TokioIo::new(MonitoredConnection::new(TokioIo::new(inner), monitor))
86 }
87}
88
89impl<C: AsyncRead, M: MonitorAsyncReadWrite> AsyncRead for MonitoredConnection<C, M> {
90 fn poll_read(
91 self: Pin<&mut Self>,
92 cx: &mut Context<'_>,
93 buf: &mut ReadBuf<'_>,
94 ) -> Poll<std::io::Result<()>> {
95 let before_buf_size = buf.filled().len();
96 let (inner, monitor) = MonitoredConnection::project_into(self);
97 let ret = inner.poll_read(cx, buf);
98 match &ret {
99 Poll::Ready(Ok(())) => {
100 let after_buf_size = buf.filled().len();
101 match after_buf_size.cmp(&before_buf_size) {
102 Ordering::Less => {
103 unreachable!(
104 "buf size decrease after poll read. Bad AsyncRead implementation on {}",
105 type_name::<C>()
106 );
107 }
108 Ordering::Equal => {
109 monitor.on_eof();
110 }
111 Ordering::Greater => {
112 monitor.on_read(after_buf_size - before_buf_size);
113 }
114 }
115 }
116 Poll::Ready(Err(e)) => {
117 monitor.on_read_err(e);
118 }
119 Poll::Pending => {}
120 }
121 ret
122 }
123}
124
125impl<C: hyper::rt::Read, M: MonitorAsyncReadWrite> hyper::rt::Read for MonitoredConnection<C, M> {
126 fn poll_read(
127 self: Pin<&mut Self>,
128 cx: &mut Context<'_>,
129 buf: hyper::rt::ReadBufCursor<'_>,
130 ) -> Poll<Result<(), std::io::Error>> {
131 hyper::rt::Read::poll_read(std::pin::pin!(self.hyper_tokio_delegate()), cx, buf)
132 }
133}
134
135impl<C: AsyncWrite, M: MonitorAsyncReadWrite> AsyncWrite for MonitoredConnection<C, M> {
136 fn poll_write(
137 self: Pin<&mut Self>,
138 cx: &mut Context<'_>,
139 buf: &[u8],
140 ) -> Poll<Result<usize, Error>> {
141 let (inner, monitor) = MonitoredConnection::project_into(self);
142 let ret = inner.poll_write(cx, buf);
143 match &ret {
144 Poll::Ready(Ok(size)) => {
145 monitor.on_write(*size);
146 }
147 Poll::Ready(Err(e)) => {
148 monitor.on_write_err(e);
149 }
150 Poll::Pending => {}
151 }
152 ret
153 }
154
155 fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
156 let (inner, monitor) = MonitoredConnection::project_into(self);
157 let ret = inner.poll_flush(cx);
158 match &ret {
159 Poll::Ready(Ok(())) => {
160 monitor.on_flush();
161 }
162 Poll::Ready(Err(e)) => {
163 monitor.on_write_err(e);
164 }
165 Poll::Pending => {}
166 }
167 ret
168 }
169
170 fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
171 let (inner, monitor) = MonitoredConnection::project_into(self);
172 let ret = inner.poll_shutdown(cx);
173 match &ret {
174 Poll::Ready(result) => {
175 monitor.on_shutdown();
176 if let Err(e) = result {
177 monitor.on_write_err(e);
178 }
179 }
180 Poll::Pending => {}
181 }
182 ret
183 }
184
185 fn poll_write_vectored(
186 self: Pin<&mut Self>,
187 cx: &mut Context<'_>,
188 bufs: &[IoSlice<'_>],
189 ) -> Poll<Result<usize, Error>> {
190 let (inner, monitor) = MonitoredConnection::project_into(self);
191 let ret = inner.poll_write_vectored(cx, bufs);
192 match &ret {
193 Poll::Ready(Ok(size)) => {
194 monitor.on_write(*size);
195 }
196 Poll::Ready(Err(e)) => {
197 monitor.on_write_err(e);
198 }
199 Poll::Pending => {}
200 }
201 ret
202 }
203
204 fn is_write_vectored(&self) -> bool {
205 self.inner.is_write_vectored()
206 }
207}
208
209impl<C: hyper::rt::Write, M: MonitorAsyncReadWrite> hyper::rt::Write for MonitoredConnection<C, M> {
210 fn poll_write(
211 self: Pin<&mut Self>,
212 cx: &mut Context<'_>,
213 buf: &[u8],
214 ) -> Poll<Result<usize, std::io::Error>> {
215 hyper::rt::Write::poll_write(std::pin::pin!(self.hyper_tokio_delegate()), cx, buf)
216 }
217
218 fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
219 hyper::rt::Write::poll_flush(std::pin::pin!(self.hyper_tokio_delegate()), cx)
220 }
221
222 fn poll_shutdown(
223 self: Pin<&mut Self>,
224 cx: &mut Context<'_>,
225 ) -> Poll<Result<(), std::io::Error>> {
226 hyper::rt::Write::poll_shutdown(std::pin::pin!(self.hyper_tokio_delegate()), cx)
227 }
228
229 fn is_write_vectored(&self) -> bool {
230 self.inner.is_write_vectored()
231 }
232
233 fn poll_write_vectored(
234 self: Pin<&mut Self>,
235 cx: &mut Context<'_>,
236 bufs: &[std::io::IoSlice<'_>],
237 ) -> Poll<Result<usize, std::io::Error>> {
238 hyper::rt::Write::poll_write_vectored(std::pin::pin!(self.hyper_tokio_delegate()), cx, bufs)
239 }
240}
241
242impl<C: Connection, M> Connection for MonitoredConnection<C, M> {
243 fn connected(&self) -> Connected {
244 self.inner.connected()
245 }
246}
247
248#[cfg(not(madsim))]
249impl<C: tonic::transport::server::Connected, M> tonic::transport::server::Connected
250 for MonitoredConnection<C, M>
251{
252 type ConnectInfo = C::ConnectInfo;
253
254 fn connect_info(&self) -> Self::ConnectInfo {
255 self.inner.connect_info()
256 }
257}
258
259pub trait MonitorNewConnection {
260 type ConnectionMonitor: MonitorAsyncReadWrite;
261
262 fn new_connection_monitor(&self, endpoint: String) -> Self::ConnectionMonitor;
263 fn on_err(&self, endpoint: String);
264}
265
266impl<C: Service<Uri>, M: MonitorNewConnection + Clone + 'static> Service<Uri>
267 for MonitoredConnection<C, M>
268where
269 C::Future: 'static,
270{
271 type Error = C::Error;
272 type Response = MonitoredConnection<C::Response, M::ConnectionMonitor>;
273
274 type Future = impl Future<Output = Result<Self::Response, Self::Error>> + 'static;
275
276 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
277 let ret = self.inner.poll_ready(cx);
278 if let Poll::Ready(Err(_)) = &ret {
279 self.monitor.on_err("<poll_ready>".to_owned());
280 }
281 ret
282 }
283
284 fn call(&mut self, uri: Uri) -> Self::Future {
285 let endpoint = format!("{:?}", uri.host());
286 let monitor = self.monitor.clone();
287 self.inner
288 .call(uri)
289 .map(move |result: Result<_, _>| match result {
290 Ok(resp) => Ok(MonitoredConnection::new(
291 resp,
292 monitor.new_connection_monitor(endpoint),
293 )),
294 Err(e) => {
295 monitor.on_err(endpoint);
296 Err(e)
297 }
298 })
299 }
300}
301
302#[cfg(not(madsim))]
303impl<Con, E, C: futures::Stream<Item = Result<Con, E>>, M: MonitorNewConnection> futures::Stream
304 for MonitoredConnection<C, M>
305where
306 Con:
307 tonic::transport::server::Connected<ConnectInfo = tonic::transport::server::TcpConnectInfo>,
308{
309 type Item = Result<MonitoredConnection<Con, M::ConnectionMonitor>, E>;
310
311 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
312 let (inner, monitor) = MonitoredConnection::project_into(self);
313 inner.poll_next(cx).map(|opt| {
314 opt.map(|result| {
315 result.map(|conn| {
316 let remote_addr = conn.connect_info().remote_addr();
317 let endpoint = remote_addr
318 .map(|remote_addr| format!("{}", remote_addr.ip()))
319 .unwrap_or("unknown".to_owned());
320 MonitoredConnection::new(conn, monitor.new_connection_monitor(endpoint))
321 })
322 })
323 })
324 }
325
326 fn size_hint(&self) -> (usize, Option<usize>) {
327 self.inner.size_hint()
328 }
329}
330
331#[derive(Clone)]
332pub struct ConnectionMetrics {
333 connection_count: IntGaugeVec,
334 connection_create_rate: IntCounterVec,
335 connection_err_rate: IntCounterVec,
336
337 read_rate: IntCounterVec,
338 reader_count: IntGaugeVec,
339
340 write_rate: IntCounterVec,
341 writer_count: IntGaugeVec,
342
343 io_err_rate: LabelGuardedIntCounterVec,
344}
345
346pub static GLOBAL_CONNECTION_METRICS: LazyLock<ConnectionMetrics> =
347 LazyLock::new(|| ConnectionMetrics::new(&GLOBAL_METRICS_REGISTRY));
348
349impl ConnectionMetrics {
350 pub fn new(registry: &Registry) -> Self {
351 let labels = ["connection_type", "uri"];
352 let connection_count = register_int_gauge_vec_with_registry!(
353 "connection_count",
354 "The number of current existing connection",
355 &labels,
356 registry,
357 )
358 .unwrap();
359
360 let connection_create_rate = register_int_counter_vec_with_registry!(
361 "connection_create_rate",
362 "Rate on creating new connection",
363 &labels,
364 registry,
365 )
366 .unwrap();
367
368 let connection_err_rate = register_int_counter_vec_with_registry!(
369 "connection_err_rate",
370 "Error rate on creating new connection",
371 &labels,
372 registry,
373 )
374 .unwrap();
375
376 let read_rate = register_int_counter_vec_with_registry!(
377 "connection_read_rate",
378 "Read rate of a connection",
379 &labels,
380 registry,
381 )
382 .unwrap();
383
384 let reader_count = register_int_gauge_vec_with_registry!(
385 "connection_reader_count",
386 "The number of current existing reader",
387 &labels,
388 registry,
389 )
390 .unwrap();
391
392 let write_rate = register_int_counter_vec_with_registry!(
393 "connection_write_rate",
394 "Write rate of a connection",
395 &labels,
396 registry,
397 )
398 .unwrap();
399
400 let writer_count = register_int_gauge_vec_with_registry!(
401 "connection_writer_count",
402 "The number of current existing writer",
403 &labels,
404 registry,
405 )
406 .unwrap();
407
408 let io_err_rate = register_guarded_int_counter_vec_with_registry!(
409 "connection_io_err_rate",
410 "IO err rate of a connection",
411 &["connection_type", "uri", "op_type", "error_kind"],
412 registry,
413 )
414 .unwrap();
415
416 Self {
417 connection_count,
418 connection_create_rate,
419 connection_err_rate,
420 read_rate,
421 reader_count,
422 write_rate,
423 writer_count,
424 io_err_rate,
425 }
426 }
427}
428
429pub struct TcpConfig {
430 pub tcp_nodelay: bool,
431 pub keepalive_duration: Option<Duration>,
432}
433
434impl Default for TcpConfig {
435 fn default() -> Self {
436 Self {
437 tcp_nodelay: true,
440 keepalive_duration: None,
441 }
442 }
443}
444
445pub fn monitor_connector<C>(
446 connector: C,
447 connection_type: impl Into<String>,
448) -> MonitoredConnection<C, MonitorNewConnectionImpl> {
449 let connection_type = connection_type.into();
450 trace!(
451 "monitoring connector {} with type {}",
452 type_name::<C>(),
453 connection_type
454 );
455 MonitoredConnection::new(connector, MonitorNewConnectionImpl { connection_type })
456}
457
458pub struct MonitoredGaiAddrs {
459 inner: Vec<SocketAddr>,
460 pos: usize,
461}
462
463impl From<GaiAddrs> for MonitoredGaiAddrs {
464 fn from(value: GaiAddrs) -> Self {
465 Self {
466 inner: value.collect_vec(),
467 pos: 0,
468 }
469 }
470}
471
472impl Iterator for MonitoredGaiAddrs {
473 type Item = SocketAddr;
474
475 fn next(&mut self) -> Option<Self::Item> {
476 let res = self.inner.get(self.pos).cloned();
477 self.pos += 1;
478 res
479 }
480}
481
482pub struct MonitoredGaiFuture {
483 name: Name,
484 inner: GaiFuture,
485}
486
487impl std::fmt::Debug for MonitoredGaiFuture {
488 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
489 f.pad("MonitoredGaiFuture")
490 }
491}
492
493impl Future for MonitoredGaiFuture {
494 type Output = Result<MonitoredGaiAddrs, Error>;
495
496 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
497 Pin::new(&mut self.inner).poll(cx).map(|res| match res {
498 Ok(addrs) => {
499 let addrs: MonitoredGaiAddrs = addrs.into();
500 trace!("resolve {} => {:?}", self.name, addrs.inner);
501 Ok(addrs)
502 }
503 Err(err) => Err(err),
504 })
505 }
506}
507
508#[derive(Clone)]
509pub struct MonitoredGaiResolver {
510 inner: GaiResolver,
511}
512
513impl std::fmt::Debug for MonitoredGaiResolver {
514 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
515 f.pad("MonitoredGaiResolver")
516 }
517}
518
519impl Default for MonitoredGaiResolver {
520 fn default() -> Self {
521 Self {
522 inner: GaiResolver::new(),
523 }
524 }
525}
526
527impl Service<Name> for MonitoredGaiResolver {
528 type Error = Error;
529 type Future = MonitoredGaiFuture;
530 type Response = MonitoredGaiAddrs;
531
532 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
533 self.inner.poll_ready(cx)
534 }
535
536 fn call(&mut self, req: Name) -> Self::Future {
537 MonitoredGaiFuture {
538 name: req.clone(),
539 inner: self.inner.call(req),
540 }
541 }
542}
543
544#[cfg_or_panic(not(madsim))]
545fn monitored_http_connector(
546 connection_type: impl Into<String>,
547 config: TcpConfig,
548) -> MonitoredConnection<HttpConnector<MonitoredGaiResolver>, MonitorNewConnectionImpl> {
549 let resolver = MonitoredGaiResolver::default();
550 let mut http = HttpConnector::new_with_resolver(resolver);
551
552 http.enforce_http(false);
553 http.set_nodelay(config.tcp_nodelay);
554 http.set_keepalive(config.keepalive_duration);
555
556 monitor_connector(http, connection_type)
557}
558
559#[cfg_or_panic(not(madsim))]
561fn configure_endpoint(endpoint: Endpoint) -> Endpoint {
562 endpoint.http2_max_header_list_size(16 * 1024 * 1024)
565}
566
567#[easy_ext::ext(EndpointExt)]
568impl Endpoint {
569 pub async fn monitored_connect(
570 mut self,
571 connection_type: impl Into<String>,
572 config: TcpConfig,
573 ) -> Result<Channel, tonic::transport::Error> {
574 #[cfg(not(madsim))]
575 {
576 self = configure_endpoint(self);
577 let connector = monitored_http_connector(connection_type, config);
578 self.connect_with_connector(connector).await
579 }
580 #[cfg(madsim)]
581 {
582 self.connect().await
583 }
584 }
585
586 #[cfg(not(madsim))]
587 pub fn monitored_connect_lazy(
588 mut self,
589 connection_type: impl Into<String>,
590 config: TcpConfig,
591 ) -> Channel {
592 self = configure_endpoint(self);
593 let connector = monitored_http_connector(connection_type, config);
594 self.connect_with_connector_lazy(connector)
595 }
596}
597
598#[cfg(not(madsim))]
599#[easy_ext::ext(RouterExt)]
600impl<L> tonic::transport::server::Router<L> {
601 pub fn monitored_serve_with_shutdown<ResBody>(
607 self,
608 listen_addr: std::net::SocketAddr,
609 connection_type: impl Into<String>,
610 config: TcpConfig,
611 signal: impl Future<Output = ()>,
612 ) -> impl Future<Output = ()>
613 where
614 L: tower_layer::Layer<tonic::service::Routes>,
615 L::Service: Service<http::Request<tonic::body::Body>, Response = http::Response<ResBody>>
616 + Clone
617 + Send
618 + 'static,
619 <<L as tower_layer::Layer<tonic::service::Routes>>::Service as Service<
620 http::Request<tonic::body::Body>,
621 >>::Future: Send + 'static,
622 <<L as tower_layer::Layer<tonic::service::Routes>>::Service as Service<
623 http::Request<tonic::body::Body>,
624 >>::Error: Into<Box<dyn std::error::Error + Send + Sync>> + Send,
625 ResBody: http_body::Body<Data = bytes::Bytes> + Send + 'static,
626 ResBody::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
627 {
628 let connection_type = connection_type.into();
629 let incoming = tonic::transport::server::TcpIncoming::bind(listen_addr)
630 .map(|incoming| {
631 incoming
632 .with_nodelay(Some(config.tcp_nodelay))
633 .with_keepalive(config.keepalive_duration)
634 })
635 .unwrap_or_else(|err| {
636 panic!(
637 "failed to bind `{connection_type}` to `{listen_addr}`: {}",
638 err.as_report()
639 )
640 });
641 let incoming =
642 MonitoredConnection::new(incoming, MonitorNewConnectionImpl { connection_type });
643
644 async move {
645 self.serve_with_incoming_shutdown(incoming, signal)
646 .await
647 .unwrap()
648 }
649 }
650}
651
652#[cfg(madsim)]
653#[easy_ext::ext(RouterExt)]
654impl<L> tonic::transport::server::Router<L> {
655 pub async fn monitored_serve_with_shutdown(
656 self,
657 listen_addr: std::net::SocketAddr,
658 connection_type: impl Into<String>,
659 config: TcpConfig,
660 signal: impl Future<Output = ()>,
661 ) {
662 self.serve_with_shutdown(listen_addr, signal).await.unwrap()
663 }
664}
665
666#[derive(Clone)]
667pub struct MonitorNewConnectionImpl {
668 connection_type: String,
669}
670
671impl MonitorNewConnection for MonitorNewConnectionImpl {
672 type ConnectionMonitor = MonitorAsyncReadWriteImpl;
673
674 fn new_connection_monitor(&self, endpoint: String) -> Self::ConnectionMonitor {
675 let labels = [self.connection_type.as_str(), endpoint.as_str()];
676 let read_rate = GLOBAL_CONNECTION_METRICS
677 .read_rate
678 .with_label_values(&labels);
679 let reader_count = GLOBAL_CONNECTION_METRICS
680 .reader_count
681 .with_label_values(&labels);
682 let write_rate = GLOBAL_CONNECTION_METRICS
683 .write_rate
684 .with_label_values(&labels);
685 let writer_count = GLOBAL_CONNECTION_METRICS
686 .writer_count
687 .with_label_values(&labels);
688 let connection_count = GLOBAL_CONNECTION_METRICS
689 .connection_count
690 .with_label_values(&labels);
691
692 GLOBAL_CONNECTION_METRICS
693 .connection_create_rate
694 .with_label_values(&labels)
695 .inc();
696
697 MonitorAsyncReadWriteImpl::new(
698 endpoint,
699 self.connection_type.clone(),
700 read_rate,
701 reader_count,
702 write_rate,
703 writer_count,
704 connection_count,
705 )
706 }
707
708 fn on_err(&self, endpoint: String) {
709 GLOBAL_CONNECTION_METRICS
710 .connection_err_rate
711 .with_label_values(&[self.connection_type.as_str(), endpoint.as_str()])
712 .inc();
713 }
714}
715
716const READ_WRITE_RATE_REPORT_INTERVAL: u64 = 1024;
717
718pub struct MonitorAsyncReadWriteImpl {
719 endpoint: String,
720 connection_type: String,
721
722 unreported_read_rate: u64,
723 read_rate: IntCounter,
724 reader_count_guard: IntGauge,
725 is_eof: bool,
726
727 unreported_write_rate: u64,
728 write_rate: IntCounter,
729 writer_count_guard: IntGauge,
730 is_shutdown: bool,
731
732 connection_count_guard: IntGauge,
733}
734
735impl MonitorAsyncReadWriteImpl {
736 pub fn new(
737 endpoint: String,
738 connection_type: String,
739 read_rate: IntCounter,
740 reader_count: IntGauge,
741 write_rate: IntCounter,
742 writer_count: IntGauge,
743 connection_count: IntGauge,
744 ) -> Self {
745 reader_count.inc();
746 writer_count.inc();
747 connection_count.inc();
748 Self {
749 endpoint,
750 connection_type,
751 unreported_read_rate: 0,
752 read_rate,
753 reader_count_guard: reader_count,
754 is_eof: false,
755 unreported_write_rate: 0,
756 write_rate,
757 writer_count_guard: writer_count,
758 is_shutdown: false,
759 connection_count_guard: connection_count,
760 }
761 }
762}
763
764impl Drop for MonitorAsyncReadWriteImpl {
765 fn drop(&mut self) {
766 if self.unreported_read_rate > 0 {
767 self.read_rate.inc_by(self.unreported_read_rate);
768 }
769 if self.unreported_write_rate > 0 {
770 self.write_rate.inc_by(self.unreported_write_rate);
771 }
772 if !self.is_eof {
773 self.reader_count_guard.dec();
774 }
775 if !self.is_shutdown {
776 self.writer_count_guard.dec();
777 }
778 self.connection_count_guard.dec();
779 }
780}
781
782impl MonitorAsyncReadWrite for MonitorAsyncReadWriteImpl {
783 fn on_read(&mut self, size: usize) {
784 self.unreported_read_rate += size as u64;
785 if self.unreported_read_rate >= READ_WRITE_RATE_REPORT_INTERVAL {
786 self.read_rate.inc_by(self.unreported_read_rate);
787 self.unreported_read_rate = 0;
788 }
789 }
790
791 fn on_eof(&mut self) {
792 if self.is_eof {
793 warn!("get eof for multiple time");
794 return;
795 }
796 self.is_eof = true;
797 self.reader_count_guard.dec();
798 }
799
800 #[allow(rw::temporary_guarded_metric)]
801 fn on_read_err(&mut self, err: &Error) {
802 GLOBAL_CONNECTION_METRICS
805 .io_err_rate
806 .with_guarded_label_values(&[
807 self.connection_type.as_str(),
808 self.endpoint.as_str(),
809 "read",
810 err.kind().to_string().as_str(),
811 ])
812 .inc();
813 }
814
815 fn on_write(&mut self, size: usize) {
816 self.unreported_write_rate += size as u64;
817 if self.unreported_write_rate >= READ_WRITE_RATE_REPORT_INTERVAL {
818 self.write_rate.inc_by(self.unreported_write_rate);
819 self.unreported_write_rate = 0;
820 }
821 }
822
823 fn on_shutdown(&mut self) {
824 if self.is_shutdown {
825 warn!("get shutdown for multiple time");
826 return;
827 }
828 self.is_shutdown = true;
829 self.writer_count_guard.dec();
830 }
831
832 #[allow(rw::temporary_guarded_metric)]
833 fn on_write_err(&mut self, err: &Error) {
834 GLOBAL_CONNECTION_METRICS
837 .io_err_rate
838 .with_guarded_label_values(&[
839 self.connection_type.as_str(),
840 self.endpoint.as_str(),
841 "write",
842 err.kind().to_string().as_str(),
843 ])
844 .inc();
845 }
846}