shell 使用 xargs 执行并发 #shell #redis

download.sh

#!/bin/bash

providers=("jingdong")
project=/home/filex/download
etc=$project/etc
cfg=$project/cfg
log=$project/log/$(date "+%Y-%m-%d_%H-%M-%S")
parallism=8

echo "start to check missing log time point"
for provider in "${providers[@]}"; do
  $project/check_missing.sh $cfg/$provider.sh
done

echo "start to download logs"
function download() {
  $project/download --info=$1 --cfg=$2 --provider=$3
  if [ $? -eq 0 ]; then
    redis-cli srem missing_$3 $1
  fi
}

export -f download
export project

for provider in "${providers[@]}"; do
  (redis-cli --eval $project/sscan.lua missing_$provider 10 | xargs -n1 -I{} -P $parallism bash -c "download {} $etc/$provider.json $provider") &
done

wait