add CI test for g3bench

.github/workflows/codecov.yml

@@ -148,3 +148,64 @@ jobs:
           name: g3proxy-all-test
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: false
+  g3bench-test:
+    name: g3bench test
+    runs-on: ubuntu-24.04
+    services:
+      httpbin:
+        image: ghcr.io/psf/httpbin:0.10.2
+        ports:
+          - 80:8080
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install capnproto libc-ares-dev libssl-dev liblua5.4-dev
+      - name: Install binutils
+        run: |
+          cargo install cargo-binutils
+      - name: Install nginx
+        run: |
+          sudo apt-get install nginx-light
+      - name: Install netcat-openbsd
+        run: |
+          sudo apt-get install netcat-openbsd
+      - name: Listen StatsD port
+        run: |
+          nc -u -l -k 127.0.0.1 8125 >/dev/null &
+      - name: Install dnsmasq
+        run: |
+          sudo apt-get install dnsmasq-base
+      - name: Backup /etc/resolv.conf
+        run: |
+          sudo cp /etc/resolv.conf /etc/resolv.conf.backup
+      - name: Edit /etc/resolv.conf
+        run: |
+          echo "nameserver 127.0.0.1" | sudo tee /etc/resolv.conf
+      - name: Run dnsmasq
+        run: |
+          sudo dnsmasq --local-service -C ${{ github.workspace }}/scripts/coverage/g3bench/dnsmasq.conf
+      - name: run unit test
+        run: |
+          ./scripts/coverage/g3bench.sh
+      - name: Restore /etc/resolv.conf
+        run: |
+          sudo mv /etc/resolv.conf.backup /etc/resolv.conf
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v5
+        with:
+          fail_ci_if_error: true
+          disable_search: true
+          files: output.lcov
+          flags: g3bench
+          name: g3bench-all-test
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: false

scripts/coverage/g3bench.sh

@@ -0,0 +1,83 @@
+#!/bin/sh
+
+set -e
+
+SCRIPTS_DIR=$(dirname "$0")
+PROJECT_DIR=$(realpath "${SCRIPTS_DIR}/../..")
+
+
+TEST_NAME="bench-ci"
+. "${SCRIPTS_DIR}/enter.sh"
+
+# build
+cargo build -p g3bench -p g3mkcert -p g3proxy -p g3proxy-ctl
+
+all_binaries=$(find target/debug/ -maxdepth 1 -type f -perm /111 | awk '{print "-object "$0}')
+
+# run the tests
+cargo test --all
+
+all_objects=$(find target/debug/deps/ -type f -perm /111 -not -name "*.so" | awk '{print "-object "$0}')
+
+# generate resource files
+"${SCRIPTS_DIR}"/g3bench/mkcert.sh
+
+# start g3proxy
+"${PROJECT_DIR}"/target/debug/g3proxy -c "${SCRIPTS_DIR}"/g3bench/g3proxy.yaml -G ${TEST_NAME} &
+PROXY_PID=$!
+
+# start nginx
+[ -d /tmp/nginx ] || mkdir /tmp/nginx
+/usr/sbin/nginx -c "${PROJECT_DIR}"/scripts/coverage/g3bench/nginx.conf
+
+# run g3bench integration tests
+
+export SSL_CERT_FILE="${SCRIPTS_DIR}/g3bench/rootCA.pem"
+
+g3bench()
+{
+	"${PROJECT_DIR}"/target/debug/g3bench --log-error 1 "$@"
+}
+
+set -x
+
+. ${SCRIPTS_DIR}/g3bench/target_dns.sh
+. ${SCRIPTS_DIR}/g3bench/target_h1.sh
+. ${SCRIPTS_DIR}/g3bench/target_h2.sh
+. ${SCRIPTS_DIR}/g3bench/target_keyless_openssl.sh
+. ${SCRIPTS_DIR}/g3bench/target_openssl.sh
+. ${SCRIPTS_DIR}/g3bench/target_rustls.sh
+
+set +x
+
+"${PROJECT_DIR}"/target/debug/g3proxy-ctl -G ${TEST_NAME} -p $PROXY_PID offline
+
+NGINX_PID=$(cat /tmp/nginx.pid)
+kill -INT $NGINX_PID
+
+
+# get all profraw files generated in each test
+profraw_files=$(find . -type f -regex ".*/${TEST_NAME}.*\.profraw")
+
+# get indexed profile data file
+cargo profdata -- merge -o "${PROF_DATA_FILE}" ${profraw_files}
+
+# report to console
+
+IGNORE_FLAGS="--ignore-filename-regex=.cargo \
+    --ignore-filename-regex=rustc \
+    --ignore-filename-regex=target/debug/build \
+    --ignore-filename-regex=g3mkcert \
+    --ignore-filename-regex=g3fcgen \
+    --ignore-filename-regex=g3proxy \
+    --ignore-filename-regex=g3tiles \
+    --ignore-filename-regex=g3keymess \
+    --ignore-filename-regex=g3iploc"
+
+echo "==== Coverage for libs ===="
+cargo cov -- report --use-color --instr-profile="${PROF_DATA_FILE}" ${IGNORE_FLAGS} --ignore-filename-regex="g3bench" ${all_binaries} ${all_objects}
+
+echo "==== Coverage for all ===="
+cargo cov -- report --use-color --instr-profile="${PROF_DATA_FILE}" ${IGNORE_FLAGS} ${all_binaries} ${all_objects}
+
+cargo cov -- export --format=lcov --instr-profile="${PROF_DATA_FILE}" ${IGNORE_FLAGS} ${all_binaries} ${all_objects} > output.lcov

scripts/coverage/g3bench/.gitignore

@@ -0,0 +1,2 @@
+
+*.pem

scripts/coverage/g3bench/dnsmasq.conf

@@ -0,0 +1,7 @@
+resolv-file=/etc/resolv.conf.backup
+
+listen-address=127.0.0.1
+bind-interfaces
+
+address=/httpbin.local/127.0.0.1
+address=/g3proxy.local/127.0.0.1

scripts/coverage/g3bench/g3proxy.yaml

@@ -0,0 +1,68 @@
+---
+
+log: journal
+
+stat:
+  target:
+    udp: 127.0.0.1:8125
+
+resolver:
+  - name: default
+    type: c-ares
+    server:
+      - 127.0.0.1
+
+escaper:
+  - name: default
+    type: direct_fixed
+    resolver: default
+    egress_net_filter:
+      default: allow
+      allow: 127.0.0.1
+
+user-group:
+  - name: default
+    static_users:
+      - name: t1
+        token:
+          salt: 4e8f8a4e37f0fa1b
+          md5: d9d963915b9815d4cc39c196c2868900
+          sha1: c28640e7b1a3d9db98187632aeba99c0cff0ffd4
+
+server:
+  - name: rss
+    type: http_rproxy
+    listen: 127.0.0.1:9443
+    escaper: default
+    enable_tls_server: true
+    global_tls_server:
+      cert_pairs:
+        certificate: httpbin.local.pem
+        private-key: httpbin.local-key.pem
+    hosts:
+      - exact_match: httpbin.local
+        upstream: 127.0.0.1:80
+        tls_server:
+          cert_pairs:
+            certificate: httpbin.local.pem
+            private-key: httpbin.local-key.pem
+  - name: http
+    type: http_proxy
+    listen: 127.0.0.1:8080
+    escaper: default
+    user-group: default
+    tls-client:
+      ca-certificate: rootCA.pem
+  - name: tls
+    type: native_tls_port
+    listen: 127.0.0.1:8443
+    server: http
+    tls_server:
+      cert_pairs:
+        certificate: g3proxy.local.pem
+        private-key: g3proxy.local-key.pem
+  - name: socks
+    type: socks_proxy
+    listen: 127.0.0.1:1080
+    escaper: default
+    user-group: default

scripts/coverage/g3bench/mkcert.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+SCRIPT_DIR=$(dirname $0)
+
+cd "${SCRIPT_DIR}"
+
+MKCERT="../../../target/debug/g3mkcert"
+
+$MKCERT --root --common-name "g3 root" --output-cert rootCA.pem --output-key rootCA-key.pem
+
+$MKCERT --tls-server --ca-cert rootCA.pem --ca-key rootCA-key.pem --host g3proxy.local --output-cert g3proxy.local.pem --output-key g3proxy.local-key.pem
+$MKCERT --tls-server --ca-cert rootCA.pem --ca-key rootCA-key.pem --host httpbin.local --output-cert httpbin.local.pem --output-key httpbin.local-key.pem

scripts/coverage/g3bench/nginx.conf

@@ -0,0 +1,34 @@
+worker_processes auto;
+worker_cpu_affinity auto;
+
+pid /tmp/nginx.pid;
+
+events {}
+
+http {
+    access_log off;
+
+    server {
+        server_name httpbin.local;
+
+        listen 2080;
+        listen [::]:2080;
+
+        listen 2443 ssl http2;
+        listen [::]:2443 ssl http2;
+
+        # requires nginx 1.25.1
+        # http2 on;
+
+        ssl_certificate httpbin.local.pem;
+        ssl_certificate_key httpbin.local-key.pem;
+
+        location / {
+                proxy_pass http://127.0.0.1:80/;
+                client_max_body_size 10m;
+                proxy_buffering off;
+                proxy_request_buffering off;
+                client_body_temp_path /tmp/nginx;
+        }
+    }
+}

scripts/coverage/g3bench/target_dns.sh

@@ -0,0 +1,18 @@
+
+# Dns over UDP, via Cloudflare Public DNS
+g3bench dns "1.1.1.1" www.example.com,A --dump-result
+
+# Dns over TCP, via Cloudflare Public DNS
+g3bench dns "1.1.1.1" --tcp www.example.com,A --dump-result
+
+# Dns over TLS, via Cloudflare Public DNS
+g3bench dns "1.1.1.1" -e dot www.example.com,A --dump-result
+
+# Dns over Https, via Cloudflare Public DNS
+g3bench dns "1.1.1.1" -e doh www.example.com,A --dump-result
+
+# Dns over Quic, via AdGuard Public DNS
+g3bench dns "94.140.14.140" -e doq www.example.com,A --dump-result
+
+# Dns over Http/3, via AdGuard Public DNS
+g3bench dns "94.140.14.140" -e doh3 www.example.com,A --dump-result

scripts/coverage/g3bench/target_h1.sh

@@ -0,0 +1,40 @@
+
+# Http
+
+test_http()
+{
+	URL=$1
+
+	g3bench h1 ${URL} --ok-status 200
+
+	g3bench h1 ${URL} -x http://t1:[email protected]:8080 --ok-status 200
+	g3bench h1 ${URL} -x http://t1:[email protected]:8080 -p --ok-status 200
+
+	g3bench h1 ${URL} -x https://t1:[email protected]:8443 --proxy-tls-ca-cert ${SSL_CERT_FILE} --ok-status 200
+	g3bench h1 ${URL} -x https://t1:[email protected]:8443 --proxy-tls-ca-cert ${SSL_CERT_FILE} -p --ok-status 200
+
+	g3bench h1 ${URL} -x socks5h://t1:[email protected]:1080 --ok-status 200
+}
+
+test_http http://httpbin.local/get
+test_http http://httpbin.local:2080/get
+
+# Https
+
+test_https()
+{
+	URL=$1
+
+	g3bench h1 ${URL} --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+
+	g3bench h1 ${URL} -x http://t1:[email protected]:8080 --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+	g3bench h1 ${URL} -x http://t1:[email protected]:8080 -p --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+
+	g3bench h1 ${URL} -x https://t1:[email protected]:8443 --proxy-tls-ca-cert ${SSL_CERT_FILE} --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+	g3bench h1 ${URL} -x https://t1:[email protected]:8443 --proxy-tls-ca-cert ${SSL_CERT_FILE} -p --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+
+	g3bench h1 ${URL} -x socks5h://t1:[email protected]:1080 --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+}
+
+test_https https://httpbin.local:9443/get
+test_https https://httpbin.local:2443/get

scripts/coverage/g3bench/target_h2.sh

@@ -0,0 +1,8 @@
+
+g3bench h2 https://httpbin.local:2443/get --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+
+g3bench h2 https://httpbin.local:2443/get -x http://t1:[email protected]:8080 --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+
+g3bench h2 https://httpbin.local:2443/get -x https://t1:[email protected]:8443 --proxy-tls-ca-cert ${SSL_CERT_FILE} --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}
+
+g3bench h2 https://httpbin.local:2443/get -x socks5h://t1:[email protected]:1080 --ok-status 200 --tls-ca-cert ${SSL_CERT_FILE}

scripts/coverage/g3bench/target_openssl.sh

@@ -0,0 +1,4 @@
+
+g3bench openssl httpbin.local:9443 --tls-ca-cert ${SSL_CERT_FILE}
+
+g3bench openssl 127.0.0.1:9443 --tls-name httpbin.local --tls-ca-cert ${SSL_CERT_FILE}

scripts/coverage/g3bench/target_rustls.sh

@@ -0,0 +1,4 @@
+
+g3bench rustls httpbin.local:9443 --tls-ca-cert ${SSL_CERT_FILE}
+
+g3bench rustls 127.0.0.1:9443 --tls-name httpbin.local --tls-ca-cert ${SSL_CERT_FILE}