allow to set CPU affinity from env var

lib/g3-runtime/src/unaided/mod.rs

@@ -100,7 +100,27 @@ impl UnaidedRuntimeConfig {
         target_os = "dragonfly",
         target_os = "netbsd",
     ))]
-    pub fn set_mapped_sched_affinity(&mut self) -> anyhow::Result<()> {
+    pub fn auto_set_sched_affinity(&mut self) -> anyhow::Result<()> {
+        use anyhow::Context;
+
+        let mut found_env_config = false;
+        for i in 0..self.thread_number_total.get() / self.thread_number_per_rt.get() {
+            let var_name = format!("WORKER_{i}_CPU_LIST");
+            if let Some(os_s) = std::env::var_os(&var_name) {
+                let Some(s) = os_s.to_str() else {
+                    return Err(anyhow!("failed to decode env var {var_name}"));
+                };
+                let mut cpu = CpuAffinity::default();
+                cpu.parse_add(s)
+                    .context(format!("invalid CPU ID list value for env var {var_name}"))?;
+                self.sched_affinity.insert(i, cpu);
+                found_env_config = true;
+            }
+        }
+        if found_env_config {
+            return Ok(());
+        }
+
         if self.thread_number_per_rt.get() != 1 {
             return Err(anyhow!(
                 "unable to set CPU affinity for multi thread worker runtime"

lib/g3-runtime/src/unaided/yaml.rs

@@ -30,7 +30,7 @@ impl UnaidedRuntimeConfig {
                 target_os = "dragonfly",
                 target_os = "netbsd",
             ))]
-            let mut set_mapped_sched_affinity = false;
+            let mut auto_set_sched_affinity = false;
 
             g3_yaml::foreach_kv(map, |k, v| match g3_yaml::key::normalize(k).as_str() {
                 "thread_number_total" | "threads_total" | "thread_number" => {
@@ -68,8 +68,8 @@ impl UnaidedRuntimeConfig {
                             config.set_sched_affinity(id, cpu);
                         }
                         Ok(())
-                    } else if let Ok(map_all) = g3_yaml::value::as_bool(v) {
-                        set_mapped_sched_affinity = map_all;
+                    } else if let Ok(enable) = g3_yaml::value::as_bool(v) {
+                        auto_set_sched_affinity = enable;
                         Ok(())
                     } else {
                         Err(anyhow!("invalid map value for key {k}"))
@@ -102,9 +102,9 @@ impl UnaidedRuntimeConfig {
                 target_os = "dragonfly",
                 target_os = "netbsd",
             ))]
-            if set_mapped_sched_affinity {
+            if auto_set_sched_affinity {
                 config
-                    .set_mapped_sched_affinity()
+                    .auto_set_sched_affinity()
                     .context("failed to set all mapped sched affinity")?;
             }
 

scripts/cpuset_split.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+import argparse
+
+
+def get_physical_core(i: int):
+    with open(f"/sys/devices/system/cpu/cpu{i}/topology/core_id", 'r') as f:
+        core = f.read()
+    return core.rstrip()
+
+
+def cpu_set_to_list(s: str):
+    # Convert CPU SET mask to CPU ID list
+    cpus = []
+    i = 0
+    for c in reversed(s):
+        n = int(c, 16)
+
+        if n & 0b0001:
+            cpus.append(str(i))
+        i += 1
+        if n & 0b0010:
+            cpus.append(str(i))
+        i += 1
+        if n & 0b0100:
+            cpus.append(str(i))
+        i += 1
+        if n & 0b1000:
+            cpus.append(str(i))
+        i += 1
+    return cpus
+
+
+def print_by_core(l: [int]):
+    core_map = {}
+    for cpu in l:
+        core = get_physical_core(cpu)
+        if core in core_map:
+            core_map[core].append(str(cpu))
+        else:
+            core_map[core] = [str(cpu)]
+    sorted_map = sorted(core_map)
+    for core in sorted_map:
+        cpus = ','.join(core_map[core])
+        print(f"CORE:{core}: {cpus}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Split a CPU SET into parts on different cores')
+    parser.add_argument('set', nargs=1, help='The input CPU SET')
+    parser.add_argument('--by-core', action='store_true', help='List CPU ID group by CORE ID')
+
+    args = parser.parse_args()
+
+    cpu_list = cpu_set_to_list(args.set[0])
+    print("LIST: {}".format(','.join(cpu_list)));
+    if args.by_core:
+        print_by_core(cpu_list)

sphinx/g3proxy/configuration/values/runtime.rst

@@ -4,6 +4,21 @@
 Runtime
 *******
 
+.. _conf_value_cpu_id_list_str:
+
+cpu id list str
+===============
+
+A string the represent a list of CPU IDs.
+
+It could be:
+
+ - A single CPU ID
+ - CPU ID range in the form `<start>-<end>`, where `start` should be less than `end`.
+ - A list of CPU ID / CPU ID range delimited by ','
+
+.. versionadded:: 1.11.3
+
 .. _conf_value_cpu_set:
 
 cpu set
@@ -17,10 +32,8 @@ The value should be one or a sequence of CPU IDs.
 
 The CPU ID valid can be:
 
- - CPU ID in usize
- - CPU ID string
- - CPU ID range string in the form `<start>-<end>`, where `start` should be less than `end`.
- - A list of CPU ID string / CPU ID range string delimited by ','
+ - usize: a single CPU ID
+ - string: :ref:`cpu id list str <conf_value_cpu_id_list_str>`
 
 .. _CPU_SET(3): https://man7.org/linux/man-pages/man3/CPU_SET.3.html
 .. _sched_setaffinity(2): https://man7.org/linux/man-pages/man2/sched_setaffinity.2.html
@@ -77,9 +90,19 @@ Set the sched affinity for each threads.
 
 For map value, the key should be the thread id starting from 0, and the value should be :ref:`cpu set <conf_value_cpu_set>`.
 
-For bool value (only if thread_number_per_runtime is set to 1):
+For bool value:
+
+* if true
+
+  - if found any `WORKER_<N>_CPU_LIST` environment variables
+
+    it will set the CPU affinity for that corresponding runtime `<N>`, the value should be :ref:`cpu id list str <conf_value_cpu_id_list_str>`.
+
+    .. versionadded:: 1.11.3
+
+  - otherwise if thread_number_per_runtime is set to 1
 
-* if true, a default CPU SET will be set for each thread, the CPU ID in the set will match the thread ID.
+    a default CPU SET will be set for each thread, the CPU ID in the set will match the thread ID.
 
 * if false, no sched affinity will be set, just as if this config option is not present.
 

sphinx/g3tiles/configuration/values/runtime.rst

@@ -4,6 +4,21 @@
 Runtime
 *******
 
+.. _conf_value_cpu_id_list_str:
+
+cpu id list str
+===============
+
+A string the represent a list of CPU IDs.
+
+It could be:
+
+ - A single CPU ID
+ - CPU ID range in the form `<start>-<end>`, where `start` should be less than `end`.
+ - A list of CPU ID / CPU ID range delimited by ','
+
+.. versionadded:: 0.3.8
+
 .. _conf_value_cpu_set:
 
 cpu set
@@ -17,10 +32,8 @@ The value should be one or a sequence of CPU IDs.
 
 The CPU ID valid can be:
 
- - CPU ID in usize
- - CPU ID string
- - CPU ID range string in the form `<start>-<end>`, where `start` should be less than `end`.
- - A list of CPU ID string / CPU ID range string delimited by ','
+ - usize: a single CPU ID
+ - string: :ref:`cpu id list str <conf_value_cpu_id_list_str>`
 
 .. _CPU_SET(3): https://man7.org/linux/man-pages/man3/CPU_SET.3.html
 .. _sched_setaffinity(2): https://man7.org/linux/man-pages/man2/sched_setaffinity.2.html
@@ -76,9 +89,19 @@ Set the sched affinity for each threads.
 
 For map value, the key should be the thread id starting from 0, and the value should be :ref:`cpu set <conf_value_cpu_set>`.
 
-For bool value (only if thread_number_per_runtime is set to 1):
+For bool value:
+
+* if true
+
+  - if found any `WORKER_<N>_CPU_LIST` environment variables
+
+    it will set the CPU affinity for that corresponding runtime `<N>`, the value should be :ref:`cpu id list str <conf_value_cpu_id_list_str>`.
+
+    .. versionadded:: 0.3.8
+
+  - otherwise if thread_number_per_runtime is set to 1
 
-* if true, a default CPU SET will be set for each thread, the CPU ID in the set will match the thread ID.
+    a default CPU SET will be set for each thread, the CPU ID in the set will match the thread ID.
 
 * if false, no sched affinity will be set, just as if this config option is not present.