improvements to profile and benchmark based on feedback

petergoldstein · Feb 11, 2025 · f51fbcc · f51fbcc
1 parent d872856
commit f51fbcc
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 23 deletions.
diff --git a/bin/benchmark b/bin/benchmark
@@ -34,8 +34,6 @@ class StringSerializer
 end
 
 dalli_url = ENV['BENCH_CACHE_URL'] || '127.0.0.1:11211'
-
-ENV['BENCH_CACHE_URL'].gsub('unix://', '') if dalli_url.include?('unix')
 bench_target = ENV['BENCH_TARGET'] || 'set'
 bench_time = (ENV['BENCH_TIME'] || 10).to_i
 bench_warmup = (ENV['BENCH_WARMUP'] || 3).to_i
@@ -44,9 +42,9 @@ payload = 'B' * bench_payload_size
 TERMINATOR = "\r\n"
 puts "yjit: #{RubyVM::YJIT.enabled?}"
 
-client = Dalli::Client.new('localhost', serializer: StringSerializer, compress: false, raw: true)
-# multi_client = Dalli::Client.new('localhost:11211,localhost:11222', serializer: StringSerializer, compress: false,
-#                                                                    raw: true)
+client = Dalli::Client.new(dalli_url, serializer: StringSerializer, compress: false, raw: true)
+multi_client = Dalli::Client.new('localhost:11211,localhost:11222', serializer: StringSerializer, compress: false,
+                                                                    raw: true)
 
 # The raw socket implementation is used to benchmark the performance of dalli & the overhead of the various abstractions
 # in the library.
@@ -56,18 +54,29 @@ sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
 # Benchmarks didn't see any performance gains from increasing the SO_RCVBUF buffer size
 # sock.setsockopt(Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, 1024 * 1024 * 8)
 # Benchamrks did see an improvement in performance when increasing the SO_SNDBUF buffer size
-sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, 1024 * 1024 * 8)
+# sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, 1024 * 1024 * 8)
 
 # ensure the clients are all connected and working
 client.set('key', payload)
+multi_client.set('multi_key', payload)
 sock.write("set sock_key 0 3600 #{payload.bytesize}\r\n")
 sock.write(payload)
 sock.write(TERMINATOR)
 sock.flush
 sock.readline # clear the buffer
 
+raise 'dalli client mismatch' if payload != client.get('key')
+
+raise 'multi dalli client mismatch' if payload != multi_client.get('multi_key')
+
+sock.write("mg sock_key v\r\n")
+sock.readline
+sock_value = sock.read(payload.bytesize)
+sock.read(TERMINATOR.bytesize)
+raise 'sock mismatch' if payload != sock_value
+
 # ensure we have basic data for the benchmarks and get calls
-payload_smaller = 'B' * 50_000
+payload_smaller = 'B' * (bench_payload_size / 10)
 pairs = {}
 100.times do |i|
   pairs["multi_#{i}"] = payload_smaller
@@ -157,17 +166,22 @@ end
 if %w[all get].include?(bench_target)
   Benchmark.ips do |x|
     x.config(warmup: bench_warmup, time: bench_time, suite: suite)
-    x.report('get dalli') { client.get('key') }
+    x.report('get dalli') do
+      result = client.get('key')
+      raise 'mismatch' unless result == payload
+    end
     # NOTE: while this is the fastest it is not thread safe and is blocking vs IO sharing friendly
     x.report('get sock') do
-      sock.write("get sock_key\r\n")
+      sock.write("mg sock_key v\r\n")
       sock.readline
-      sock.read(payload.bytesize)
+      result = sock.read(payload.bytesize)
+      sock.read(TERMINATOR.bytesize)
+      raise 'mismatch' unless result == payload
     end
     # NOTE: This shows that when adding thread safety & non-blocking IO we are slower for single process/thread use case
     x.report('get sock non-blocking') do
       @lock.synchronize do
-        sock.write("get sock_key\r\n")
+        sock.write("mg sock_key v\r\n")
         sock.readline
         count = payload.bytesize
         value = String.new(capacity: count + 1)
@@ -183,6 +197,8 @@ if %w[all get].include?(bench_target)
           end
           break if value.bytesize == count
         end
+        sock.read(TERMINATOR.bytesize)
+        raise 'mismatch' unless value == payload
       end
     end
     x.compare!
@@ -192,8 +208,14 @@ end
 if %w[all get_multi].include?(bench_target)
   Benchmark.ips do |x|
     x.config(warmup: bench_warmup, time: bench_time, suite: suite)
-    x.report('get 100 keys') { client.get_multi(pairs.keys) }
-    x.report('get 100 keys raw sock') { sock_get_multi(sock, pairs) }
+    x.report('get 100 keys') do
+      result = client.get_multi(pairs.keys)
+      raise 'mismatch' unless result == pairs
+    end
+    x.report('get 100 keys raw sock') do
+      result = sock_get_multi(sock, pairs)
+      raise 'mismatch' unless result == pairs
+    end
     x.compare!
   end
 end

diff --git a/bin/profile b/bin/profile
@@ -34,15 +34,13 @@ class StringSerializer
 end
 
 dalli_url = ENV['BENCH_CACHE_URL'] || '127.0.0.1:11211'
-
-ENV['BENCH_CACHE_URL'].gsub('unix://', '') if dalli_url.include?('unix')
 bench_target = ENV['BENCH_TARGET'] || 'get'
 bench_time = (ENV['BENCH_TIME'] || 10).to_i
 bench_payload_size = (ENV['BENCH_PAYLOAD_SIZE'] || 700_000).to_i
 TERMINATOR = "\r\n"
 puts "yjit: #{RubyVM::YJIT.enabled?}"
 
-client = Dalli::Client.new('localhost', serializer: StringSerializer, compress: false)
+client = Dalli::Client.new(dalli_url, serializer: StringSerializer, compress: false)
 
 # The raw socket implementation is used to benchmark the performance of dalli & the overhead of the various abstractions
 # in the library.
@@ -52,7 +50,7 @@ sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
 # Benchmarks didn't see any performance gains from increasing the SO_RCVBUF buffer size
 # sock.setsockopt(Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, 1024 * 1024 * 8)
 # Benchamrks did see an improvement in performance when increasing the SO_SNDBUF buffer size
-sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, 1024 * 1024 * 8)
+# sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, 1024 * 1024 * 8)
 
 payload = 'B' * bench_payload_size
 dalli_key = 'dalli_key'
@@ -65,7 +63,7 @@ sock.flush
 sock.readline # clear the buffer
 
 # ensure we have basic data for the benchmarks and get calls
-payload_smaller = 'B' * 50_000
+payload_smaller = 'B' * (bench_payload_size / 10)
 pairs = {}
 100.times do |i|
   pairs["multi_#{i}"] = payload_smaller
@@ -128,15 +126,20 @@ end
 if %w[all get].include?(bench_target)
   Vernier.profile(out: 'client_get_profile.json') do
     start_time = Time.now
-    client.get(dalli_key) while Time.now - start_time < bench_time
+    while Time.now - start_time < bench_time
+      result = client.get(dalli_key)
+      raise 'mismatch' unless result == payload
+    end
   end
 
   Vernier.profile(out: 'socket_get_profile.json') do
     start_time = Time.now
     while Time.now - start_time < bench_time
-      sock.write("get sock_key\r\n")
+      sock.write("mg sock_key v\r\n")
       sock.readline
-      sock.read(payload.bytesize)
+      result = sock.read(payload.bytesize)
+      sock.read(TERMINATOR.bytesize)
+      raise 'mismatch' unless result == payload
     end
   end
 end
@@ -162,12 +165,18 @@ end
 if %w[all get_multi].include?(bench_target)
   Vernier.profile(out: 'client_get_multi_profile.json') do
     start_time = Time.now
-    client.get_multi(pairs.keys) while Time.now - start_time < bench_time
+    while Time.now - start_time < bench_time
+      result = client.get_multi(pairs.keys)
+      raise 'mismatch' unless result == pairs
+    end
   end
 
   Vernier.profile(out: 'socket_get_multi_profile.json') do
     start_time = Time.now
-    sock_get_multi(sock, pairs) while Time.now - start_time < bench_time
+    while Time.now - start_time < bench_time
+      result = sock_get_multi(sock, pairs)
+      raise 'mismatch' unless result == pairs
+    end
   end
 end