diff --git a/g3proxy/src/config/escaper/direct_fixed.rs b/g3proxy/src/config/escaper/direct_fixed.rs index 95c06e3b1..4ab597aab 100644 --- a/g3proxy/src/config/escaper/direct_fixed.rs +++ b/g3proxy/src/config/escaper/direct_fixed.rs @@ -25,7 +25,9 @@ use g3_types::acl::{AclAction, AclNetworkRuleBuilder}; use g3_types::metrics::{NodeName, StaticMetricsTags}; #[cfg(any(target_os = "linux", target_os = "android"))] use g3_types::net::InterfaceName; -use g3_types::net::{HappyEyeballsConfig, TcpKeepAliveConfig, TcpMiscSockOpts, UdpMiscSockOpts}; +use g3_types::net::{ + HappyEyeballsConfig, ProxyProtocolVersion, TcpKeepAliveConfig, TcpMiscSockOpts, UdpMiscSockOpts, +}; use g3_types::resolve::{QueryStrategy, ResolveRedirectionBuilder, ResolveStrategy}; use g3_yaml::YamlDocPosition; @@ -54,6 +56,7 @@ pub(crate) struct DirectFixedEscaperConfig { pub(crate) tcp_misc_opts: TcpMiscSockOpts, pub(crate) udp_misc_opts: UdpMiscSockOpts, pub(crate) enable_path_selection: bool, + pub(crate) use_proxy_protocol: Option, pub(crate) extra_metrics_tags: Option>, } @@ -79,6 +82,7 @@ impl DirectFixedEscaperConfig { tcp_misc_opts: Default::default(), udp_misc_opts: Default::default(), enable_path_selection: false, + use_proxy_protocol: None, extra_metrics_tags: None, } } @@ -195,6 +199,12 @@ impl DirectFixedEscaperConfig { .context(format!("invalid happy eyeballs config value for key {k}"))?; Ok(()) } + "use_proxy_protocol" => { + let version = g3_yaml::value::as_proxy_protocol_version(v) + .context(format!("invalid ProxyProtocolVersion value for key {k}"))?; + self.use_proxy_protocol = Some(version); + Ok(()) + } _ => Err(anyhow!("invalid key {k}")), } } diff --git a/g3proxy/src/escape/direct_fixed/ftp_connect/mod.rs b/g3proxy/src/escape/direct_fixed/ftp_connect/mod.rs index 07ffd0f45..f243ea428 100644 --- a/g3proxy/src/escape/direct_fixed/ftp_connect/mod.rs +++ b/g3proxy/src/escape/direct_fixed/ftp_connect/mod.rs @@ -35,9 +35,13 @@ impl DirectFixedEscaper { task_notes: &ServerTaskNotes, task_stats: ArcFtpTaskRemoteControlStats, ) -> Result { - let stream = self + let mut stream = self .tcp_connect_to(task_conf, tcp_notes, task_notes) .await?; + if let Some(version) = self.config.use_proxy_protocol { + self.send_tcp_proxy_protocol_header(version, &mut stream, task_notes, false) + .await?; + } let mut wrapper_stats = FtpControlRemoteWrapperStats::new(&self.stats, task_stats); wrapper_stats.push_user_io_stats(self.fetch_user_upstream_io_stats(task_notes)); diff --git a/g3proxy/src/escape/direct_fixed/http_forward/mod.rs b/g3proxy/src/escape/direct_fixed/http_forward/mod.rs index 2e5e5af5a..1f24ccbf3 100644 --- a/g3proxy/src/escape/direct_fixed/http_forward/mod.rs +++ b/g3proxy/src/escape/direct_fixed/http_forward/mod.rs @@ -43,9 +43,13 @@ impl DirectFixedEscaper { task_notes: &ServerTaskNotes, task_stats: ArcHttpForwardTaskRemoteStats, ) -> Result { - let stream = self + let mut stream = self .tcp_connect_to(task_conf, tcp_notes, task_notes) .await?; + if let Some(version) = self.config.use_proxy_protocol { + self.send_tcp_proxy_protocol_header(version, &mut stream, task_notes, false) + .await?; + } let (ups_r, ups_w) = stream.into_split(); diff --git a/g3proxy/src/escape/direct_fixed/mod.rs b/g3proxy/src/escape/direct_fixed/mod.rs index ca0e9f30b..6e0d8066f 100644 --- a/g3proxy/src/escape/direct_fixed/mod.rs +++ b/g3proxy/src/escape/direct_fixed/mod.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; use slog::Logger; +use tokio::io::{AsyncWrite, AsyncWriteExt}; use g3_daemon::stat::remote::ArcTcpConnectionTaskRemoteStats; use g3_resolver::ResolveError; @@ -28,7 +29,7 @@ use g3_socket::BindAddr; use g3_socket::util::AddressFamily; use g3_types::acl::AclNetworkRule; use g3_types::metrics::NodeName; -use g3_types::net::{Host, UpstreamAddr}; +use g3_types::net::{Host, ProxyProtocolEncoder, ProxyProtocolVersion, UpstreamAddr}; use g3_types::resolve::{ResolveRedirection, ResolveStrategy}; use super::{ @@ -255,6 +256,34 @@ impl DirectFixedEscaper { } } + async fn send_tcp_proxy_protocol_header( + &self, + version: ProxyProtocolVersion, + writer: &mut W, + task_notes: &ServerTaskNotes, + do_flush: bool, + ) -> Result<(), TcpConnectError> + where + W: AsyncWrite + Unpin, + { + let mut encoder = ProxyProtocolEncoder::new(version); + let bytes = encoder + .encode_tcp(task_notes.client_addr(), task_notes.server_addr()) + .map_err(TcpConnectError::ProxyProtocolEncodeError)?; + writer + .write_all(bytes) // no need to flush data + .await + .map_err(TcpConnectError::ProxyProtocolWriteFailed)?; + self.stats.tcp.io.add_out_bytes(bytes.len() as u64); + if do_flush { + writer + .flush() + .await + .map_err(TcpConnectError::ProxyProtocolWriteFailed)?; + } + Ok(()) + } + fn fetch_user_upstream_io_stats( &self, task_notes: &ServerTaskNotes, diff --git a/g3proxy/src/escape/direct_fixed/tcp_connect/mod.rs b/g3proxy/src/escape/direct_fixed/tcp_connect/mod.rs index 551fefc09..9aa16c45f 100644 --- a/g3proxy/src/escape/direct_fixed/tcp_connect/mod.rs +++ b/g3proxy/src/escape/direct_fixed/tcp_connect/mod.rs @@ -455,9 +455,13 @@ impl DirectFixedEscaper { task_notes: &ServerTaskNotes, task_stats: ArcTcpConnectionTaskRemoteStats, ) -> TcpConnectResult { - let stream = self + let mut stream = self .tcp_connect_to(task_conf, tcp_notes, task_notes) .await?; + if let Some(version) = self.config.use_proxy_protocol { + self.send_tcp_proxy_protocol_header(version, &mut stream, task_notes, true) + .await?; + } let (r, w) = stream.into_split(); let mut wrapper_stats = TcpConnectRemoteWrapperStats::new(&self.stats, task_stats); diff --git a/g3proxy/src/escape/direct_fixed/tls_connect/mod.rs b/g3proxy/src/escape/direct_fixed/tls_connect/mod.rs index 0798053d2..baa7a6608 100644 --- a/g3proxy/src/escape/direct_fixed/tls_connect/mod.rs +++ b/g3proxy/src/escape/direct_fixed/tls_connect/mod.rs @@ -40,9 +40,13 @@ impl DirectFixedEscaper { task_notes: &ServerTaskNotes, tls_application: TlsApplication, ) -> Result>, TcpConnectError> { - let stream = self + let mut stream = self .tcp_connect_to(&task_conf.tcp, tcp_notes, task_notes) .await?; + if let Some(version) = self.config.use_proxy_protocol { + self.send_tcp_proxy_protocol_header(version, &mut stream, task_notes, false) + .await?; + } // set limit config and add escaper stats, do not count in task stats let limit_config = &self.config.general.tcp_sock_speed_limit; diff --git a/scripts/coverage/g3proxy/0003_chain_tcp_stream/g3proxy.yaml b/scripts/coverage/g3proxy/0003_chain_tcp_stream/g3proxy.yaml new file mode 100644 index 000000000..333bee795 --- /dev/null +++ b/scripts/coverage/g3proxy/0003_chain_tcp_stream/g3proxy.yaml @@ -0,0 +1,53 @@ +--- + +log: discard + +stat: + target: + udp: 127.0.0.1:8125 + +resolver: + - name: default + type: c-ares + server: + - 127.0.0.1 + +escaper: + - name: default + type: direct_fixed + resolver: default + egress_net_filter: + default: allow + allow: 127.0.0.1 + - name: to_inner + type: direct_fixed + resolver: default + egress_net_filter: + default: allow + allow: 127.0.0.1 + use_proxy_protocol: 2 + +server: + - name: tcp + type: tcp_stream + listen: 127.0.0.1:8080 + escaper: to_inner + upstream: 127.0.0.1:8081 + - name: tls + type: tls_stream + escaper: to_inner + listen: 127.0.0.1:8443 + tls_server: + cert_pairs: + certificate: ../httpbin.local.pem + private-key: ../httpbin.local-key.pem + upstream: 127.0.0.1:8081 + - name: inner_tcp_server + type: tcp_stream + escaper: default + upstream: 127.0.0.1:80 + - name: inner_tcp_port + type: plain_tcp_port + listen: 127.0.0.1:8081 + server: inner_tcp_server + proxy_protocol: 2 diff --git a/scripts/coverage/g3proxy/0003_chain_tcp_stream/testcases.sh b/scripts/coverage/g3proxy/0003_chain_tcp_stream/testcases.sh new file mode 100644 index 000000000..5e6810cb7 --- /dev/null +++ b/scripts/coverage/g3proxy/0003_chain_tcp_stream/testcases.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -T http://httpbin.local:8080 + +python3 "${PROJECT_DIR}/g3proxy/ci/python3+curl/test_httpbin.py" -T https://httpbin.local:8443 --ca-cert "${TEST_CA_CERT_FILE}" diff --git a/sphinx/g3proxy/configuration/escapers/direct_fixed.rst b/sphinx/g3proxy/configuration/escapers/direct_fixed.rst index 4d091481c..6e42a97c5 100644 --- a/sphinx/g3proxy/configuration/escapers/direct_fixed.rst +++ b/sphinx/g3proxy/configuration/escapers/direct_fixed.rst @@ -92,3 +92,14 @@ Weather we should enable path selection. .. note:: Path selection on server side should be open, or this option will have no effects. **default**: false + +use_proxy_protocol +------------------ + +**optional**, **type**: :ref:`proxy protocol version ` + +Set the version of PROXY protocol we use for outgoing tcp connections except for FTP data connections. + +**default**: not set, which means PROXY protocol won't be used + +.. versionadded:: 1.11.3 diff --git a/sphinx/g3proxy/configuration/escapers/index.rst b/sphinx/g3proxy/configuration/escapers/index.rst index d7359cb9a..4b66c3e02 100644 --- a/sphinx/g3proxy/configuration/escapers/index.rst +++ b/sphinx/g3proxy/configuration/escapers/index.rst @@ -225,17 +225,6 @@ to the real username, the password field set to our package name (g3proxy if not .. note:: This will conflict with the real auth of next proxy. -.. _conf_escaper_common_use_proxy_protocol: - -use_proxy_protocol ------------------- - -**optional**, **type**: :ref:`proxy protocol version ` - -Set the version of PROXY protocol we use for outgoing tcp connections. - -**default**: not set, which means PROXY protocol won't be used - .. _conf_escaper_common_peer_negotiation_timeout: peer_negotiation_timeout diff --git a/sphinx/g3proxy/configuration/escapers/proxy_http.rst b/sphinx/g3proxy/configuration/escapers/proxy_http.rst index d1b22b80e..29a5afe60 100644 --- a/sphinx/g3proxy/configuration/escapers/proxy_http.rst +++ b/sphinx/g3proxy/configuration/escapers/proxy_http.rst @@ -25,7 +25,6 @@ The following common keys are supported: * :ref:`happy eyeballs ` * :ref:`tcp_misc_opts ` * :ref:`pass_proxy_userid ` -* :ref:`use_proxy_protocol ` * :ref:`peer negotiation timeout ` * :ref:`extra_metrics_tags ` @@ -113,3 +112,12 @@ Set tcp keepalive. The tcp keepalive set in user config won't be taken into account. **default**: no keepalive set + +use_proxy_protocol +------------------ + +**optional**, **type**: :ref:`proxy protocol version ` + +Set the version of PROXY protocol to use after TCP connected to the peer. + +**default**: not set, which means PROXY protocol won't be used diff --git a/sphinx/g3proxy/configuration/escapers/proxy_https.rst b/sphinx/g3proxy/configuration/escapers/proxy_https.rst index 8570114fc..f2744945f 100644 --- a/sphinx/g3proxy/configuration/escapers/proxy_https.rst +++ b/sphinx/g3proxy/configuration/escapers/proxy_https.rst @@ -25,7 +25,6 @@ The following common keys are supported: * :ref:`happy eyeballs ` * :ref:`tcp_misc_opts ` * :ref:`pass_proxy_userid ` -* :ref:`use_proxy_protocol ` * :ref:`peer negotiation timeout ` * :ref:`extra_metrics_tags ` @@ -132,3 +131,12 @@ Set tcp keepalive. The tcp keepalive set in user config won't be taken into account. **default**: no keepalive set + +use_proxy_protocol +------------------ + +**optional**, **type**: :ref:`proxy protocol version ` + +Set the version of PROXY protocol to use after TCP connected to the peer. + +**default**: not set, which means PROXY protocol won't be used