Conflux (aka Conslidated Crawler/Pipeline) Scripts

kill.sh

#!/usr/local/bin/bash

pgrep -f rosetta | xargs kill
pgrep -f node | xargs kill -9
pgrep -f rake | xargs kill -9
sleep 10
for proc in node rake rosetta ;do
    echo Checking for $proc processes
    pgrep -f $proc
done

start.sh

#!/usr/local/bin/bash

if [ ! -d logs ]; then
    mkdir logs
fi

cd ./rosetta

echo sendToScout/rosetta
PROCESS=sendToScout rosetta > ../logs/rosetta-in.logs 2>&1 &

cd ../scout

echo exchange:director
nohup rake exchange:director > ../logs/director.log 2>&1 &
sleep 1

echo proxy:interceptor
nohup rake proxy:interceptor > ../logs/proxy.log 2>&1 &
sleep 1

echo micro:product:real_time:consumer
nohup rake micro:product:real_time:consumer > ../logs/product.log 2>&1 &
sleep 1

echo micro:marketplace:real_time:consumer
nohup rake micro:marketplace:real_time:consumer > ../logs/marketplace.log  2>&1 &
sleep 1

echo exchange:post_director
nohup rake exchange:post_director > ../logs/post.log 2>&1 &
sleep 1

cd ../premo

echo premo curl
PREMO_ENGINE=curl PORT=3000 yarn start > ../logs/curl.log 2>&1 &
sleep 1

echo premo puppeteer
PREMO_ENGINE=puppeteer PORT=3001 yarn start > ../logs/puppeteer.log 2>&1 &
sleep 1

echo premo nightmare
PREMO_ENGINE=nightmare PORT=3002 yarn start > ../logs/nightmare.log 2>&1 &
sleep 1

cd ../rosetta
echo sentToPS/rosetta
PROCESS=sendToPS rosetta > ../logs/rosetta-out.logs 2>&1 &
cd ..

echo ===== Pipeline active =====

Sourced from https://github.com/PriceSpider-NeuIntel/harpoon