Created
March 5, 2019 11:32
-
-
Save sahilbadyal/3990fe16712cd670e6b39460960e6377 to your computer and use it in GitHub Desktop.
Assumes SSH access allowed between nodes of cluster
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
##Author: Sahil Badyal <[email protected]> | |
usage() | |
{ | |
echo "usage: sysinfo_page [[[-f file ] [-i]] | [-h]]" | |
} | |
if [ "1" == "" ]; then | |
usage | |
exit | |
fi | |
##Main | |
## bash run_distributed.sh -o <o path> -i <i path> -bs 512 -s 800 -e 4000 -f 477 | |
while [ "$1" != "" ]; do | |
case $1 in | |
-o | --output ) shift | |
out_dir=$1 | |
;; | |
-i | --input ) shift | |
input_dir=$1 | |
;; | |
-f | --feature_lene) shift | |
flen=$1 | |
;; | |
-bs | --batch-size ) shift | |
batch_size=$1 | |
;; | |
-s | --save-every ) shift | |
save=$1 | |
;; | |
-e | --eval-every ) shift | |
eval_=$1 | |
;; | |
-pb | --prefetch-buffer ) shift | |
pb=$1 | |
;; | |
-sb | --shuffle-buffer ) shift | |
sb=$1 | |
;; | |
-h | --help ) usage | |
exit | |
;; | |
* ) usage | |
exit 1 | |
esac | |
shift | |
done | |
chief=$(echo $TF_CONFIG | jq '.cluster' | jq '.chief' | jq '.[]' | sed 's/:2222//g' | tr -d '"') | |
ips=$(echo $TF_CONFIG | jq '.cluster' | jq '.worker' | jq '.[]' | sed 's/:2222//g' | tr -d '"') | |
tot_workers=$(echo $TF_CONFIG | jq '.cluster' | jq '.worker' | jq '.[]' | wc -l) | |
tot_workers=$((tot_workers+1)) | |
ips_ps=$(echo $TF_CONFIG | jq '.cluster' | jq '.ps' | jq '.[]' | sed 's/:2222//g' | tr -d '"') | |
read -a host <<< $ips | |
read -a ps <<< $ips_ps | |
k=0 | |
python <path to file> -rdb -bs $batch_size -i $input_dir -pb $pb -sb $sb -o $out_dir -fl $flen -s $save -e $eval_ -tw $tot_workers -wi $k > /dev/null 2>&1 & | |
for x in "${ps[@]}" | |
do | |
com="python <path to file> -rdb -bs $batch_size -i $input_dir -pb $pb -sb $sb -o $out_dir -fl $flen -s $save -e $eval_ -tw $tot_workers -wi $k > /dev/null 2>&1 &" | |
ssh -o "StrictHostKeyChecking no" hadoop@$x $com | |
echo $com | |
done | |
k=$((k+1)) | |
for x in "${host[@]}" | |
do | |
com="python <path to file> -rdb -i $input_dir -bs $batch_size -pb $pb -sb $sb -o $out_dir -fl $flen -s $save -e $eval_ -tw $tot_workers -wi $k > /dev/null 2>&1 &" | |
ssh -o "StrictHostKeyChecking no" hadoop@$x $com | |
k=$((k+1)) | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment