issue 3833 ability to sync files in parallel in sync_to_storage (#3855)

workflows/pipe-common/shell/sync_to_storage

@@ -59,6 +59,17 @@ function remove_from_spec() {
     fi
 }
 
+function wait_for_pids() {
+    local _sync_pids=("$@")
+    for pid in ${_sync_pids[*]}; do
+        echo_debug "Waiting for pid $pid to finish"
+        wait "$pid"
+        if [ $? -ne 0 ]; then
+            echo "[WARN] Sync process with pid $pid failed"
+        fi
+    done
+}
+
 function start_sync_daemon() {
     while [ -z "$CP_SYNC_TO_STORAGE_STOP" ]; do
 
@@ -75,6 +86,7 @@ function start_sync_daemon() {
             continue
         fi
 
+        local _sync_pids=()
         while read -a _sync_config_entry; do
             echo_debug "Processing entry ${_sync_config_entry[*]}"
 
@@ -115,13 +127,29 @@ function start_sync_daemon() {
 
             local _pipe_command="pipe storage cp -f \"$_source_path\" \"$_dest_path\" $_pipe_extra_args"
             echo_debug "pipe command: $_pipe_command"
-            eval "$_pipe_command"
 
-            if [ $? -ne 0 ]; then
-                echo "[WARN] Sync $_source_path to $_dest_path failed"
+            if [ "$CP_SYNC_TO_STORAGE_THREADS" -gt 1 ]; then
+                eval "$_pipe_command" &
+                _sync_process_pid=$!
+                echo_debug "Sync $_source_path to $_dest_path has pid $_sync_process_pid"
+
+                _sync_pids[${#_sync_pids[@]}]=$_sync_process_pid
+                if [ "${#_sync_pids[@]}" -eq "$CP_SYNC_TO_STORAGE_THREADS" ]; then
+                  wait_for_pids "${_sync_pids[*]}"
+                  _sync_pids=()
+                fi
+            else
+                eval "$_pipe_command"
+                if [ $? -ne 0 ]; then
+                    echo "[WARN] Sync $_source_path to $_dest_path failed"
+                fi
             fi
         done <"$CP_SYNC_TO_STORAGE_SPEC"
 
+        if [ "${#_sync_pids[@]}" -gt 0 ]; then
+            wait_for_pids "${_sync_pids[*]}"
+        fi
+
         # Check if some file from config marked for deletion
         if [ -f "$CP_SYNC_TO_STORAGE_REMOVE_SPEC" ]; then
             while read -a _sync_config_entry; do
@@ -151,6 +179,7 @@ export CP_SYNC_TO_STORAGE_REMOVE_SPEC="${CP_SYNC_TO_STORAGE_REMOVE_SPEC:-/etc/sy
 export CP_SYNC_TO_STORAGE_PID_FILE="${CP_SYNC_TO_STORAGE_PID_FILE:-/var/run/sync_to_storage.pid}"
 export CP_SYNC_TO_STORAGE_STOP_FLAG_FILE="${CP_SYNC_TO_STORAGE_STOP_FLAG_FILE:-/var/run/sync_to_storage.stop}"
 export CP_SYNC_TO_STORAGE_TIMEOUT_SEC="${CP_SYNC_TO_STORAGE_TIMEOUT_SEC:-3600}"
+export CP_SYNC_TO_STORAGE_THREADS="${CP_SYNC_TO_STORAGE_THREADS:-1}"
 export CP_SYNC_TO_STORAGE_LOGFILE="${CP_SYNC_TO_STORAGE_LOGFILE:-/var/log/sync_to_storage.log}"
 export CP_SYNC_TO_STORAGE_DEBUG="${CP_SYNC_TO_STORAGE_DEBUG:-false}"
 
@@ -185,6 +214,7 @@ elif [ "$_cmd" == "start" ]; then
         rm -f "$CP_SYNC_TO_STORAGE_PID_FILE"
     fi
     export -f start_sync_daemon
+    export -f wait_for_pids
     export -f echo_debug
     nohup bash -c start_sync_daemon &> "$CP_SYNC_TO_STORAGE_LOGFILE" &
     echo "$!" > "$CP_SYNC_TO_STORAGE_PID_FILE"