fcrepo4 clustering and SGE

example.sh

# launch a 4 node cluster
qsub -t 1-4 -cwd fcrepo-test.sh

# run a 1 node stress test against the cluster launched in job id 1
qsub -v CLUSTER_JOB_ID=1 -cwd fcrepo-stress-test.sh

fcrepo-cluster-launcher.sh

#!/bin/bash


export RUN_HOURS=${RUN_TIME:-24};
if [ -z "$JOB_ID" ]; then
  export JOB_ID=0;
  export SGE_TASK_ID=0;
  export SGE_TASK_FIRST=0;
  export SGE_TASK_LAST=0;
fi

export BASE_DIR=/mnt/glusterfs/`whoami`
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export JAVA_BIN=$JAVA_HOME/jre/bin/java
export PROJECT_DIR=$BASE_DIR/fcrepo4-testing
export MVN_BIN=$PROJECT_DIR/apache-maven-3.1.0-alpha-1/bin/mvn

hostname
env
$JAVA_BIN -version

if [ ! -d $PROJECT_DIR ]; then
  mkdir $PROJECT_DIR;
fi

export TASK_DIR=$PROJECT_DIR/$JOB_ID
if [ ! -d $TASK_DIR ]; then
  mkdir $TASK_DIR;
fi

echo "Using task dir $TASK_DIR"

cd $PROJECT_DIR;

# use an exclusive lock for building
if [ ! -d $PROJECT_DIR/fcrepo4/target ]; then
echo "need to build fcrepo first. obtaining a lock."
(
flock -e 200
if [ ! -d apache-maven-3.1.0-alpha-1 ]; then
  curl -o maven.tar.gz http://www.trieuvan.com/apache/maven/maven-3/3.1.0-alpha-1/binaries/apache-maven-3.1.0-alpha-1-bin.tar.gz && tar -xzf maven.tar.gz
fi

if [ ! -d fcrepo4 ]; then 
  git clone git://github.com/futures/fcrepo4.git
fi

export FCREPO_HOME=`pwd`/fcrepo4

cd $FCREPO_HOME

if [ ! -d target ]; then
  $MVN_BIN  -s $BASE_DIR/settings.xml clean install
fi
) 200>$PROJECT_DIR/build.lock
fi

cat /dev/null > $TASK_DIR/test.log
cat /dev/null > $TASK_DIR/curl.log

hostname >> $TASK_DIR/test.log

cp -R $PROJECT_DIR/fcrepo4/fcrepo-webapp $TASK_DIR/fcrepo-webapp-$JOB_ID-$SGE_TASK_ID
cd $TASK_DIR/fcrepo-webapp-$JOB_ID-$SGE_TASK_ID
echo "Copied webapp to $TASK_DIR/fcrepo-webapp-$JOB_ID-$SGE_TASK_ID"

# MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M" $MVN_BIN -s $BASE_DIR/settings.xml clean install -DskipTests=true

mkdir $TASK_DIR/jgroups

if [ $SGE_TASK_FIRST != $SGE_TASK_ID ]; then
  echo "We're not the first job. Give it some time to spin up"
  sleep 90;
fi

export JETTY_PORT=$((18080 + $SGE_TASK_ID))
echo "Starting fcrepo4 on port $JETTY_PORT"
export MAVEN_OPTS="-Xmx3g -XX:MaxPermSize=512M -Djetty.port=$JETTY_PORT -Dfcrepo.ispn.jgroups.configuration=${JGROUPS_CONFIG:-jgroups-udp.xml} -Dfcrepo.infinispan.cache_configuration=${ISPN_CONFIG:-$PROJECT_DIR/configurations/infinispan-ccl.xml} -Dfcrepo.modeshape.configuration=${MODE_CONFIG:-file:$PROJECT_DIR/configurations/repository.json} -Djava.net.preferIPv4Stack=true"
timeout -k $((60*60*$RUN_HOURS + 120))s $((60 * 60 * $RUN_HOURS))s $MVN_BIN -s $BASE_DIR/settings.xml jetty:run-war &> jetty.log &
wait_start=0
while [ $wait_start -le 60 ]; do
  echo "try $wait_start; GET http://`hostname`:$JETTY_PORT/rest/"
  
  curl -X GET "http://`hostname`:$JETTY_PORT/rest/" &> /dev/null

  if [ $? == 0 ]; then 
    wait_start=100
  fi

  wait_start=$(( $wait_start + 1 ))
  sleep 3
done
(
flock -e 201
echo "`hostname`:$JETTY_PORT" >> $TASK_DIR/hosts
) 201>$TASK_DIR/hosts.lock
pid=$!

trap "{ kill $pid;}" EXIT SIGINT SIGTERM

wait $pid

fcrepo-stress-test.sh

#!/bin/bash

export RUN_COUNT=${object_count:-1000000};
if [ -z "$JOB_ID" ]; then
  export JOB_ID=0;
  export SGE_TASK_ID=0;
  export SGE_TASK_FIRST=0;
  export SGE_TASK_LAST=0;
fi

export BASE_DIR=/mnt/glusterfs/`whoami`
export PROJECT_DIR=$BASE_DIR/fcrepo4-testing

hostname
env

if [ -z "$CLUSTER_JOB_ID" ]; then
  echo "CLUSTER_JOB_ID not defined. aborting."
  exit 1
fi

if [ ! -d $PROJECT_DIR ]; then
  mkdir $PROJECT_DIR;
fi

export TASK_DIR=$PROJECT_DIR/$CLUSTER_JOB_ID
if [ ! -d $TASK_DIR ]; then
  mkdir $TASK_DIR;
fi

echo "Using task dir $TASK_DIR"

cd $PROJECT_DIR;

while [ ! -f $TASK_DIR/hosts ]; do
  echo "Waiting for $TASK_DIR/hosts"
  sleep 5
done

sleep 5

first_host=$(head -n 1 $TASK_DIR/hosts );
date
echo "Waiting for fcrepo start.."
waitx=1
while [ $waitx -le 60 ]; do
  echo "try $waitx; GET http://$first_host/rest/"
  
  curl -X GET "http://$first_host/rest/" &> /dev/null

  if [ $? == 0 ]; then 
    waitx=100
  fi

  waitx=$(( $waitx + 1 ))
  sleep 3
done

curl -X GET "http://$first_host/rest/"

  if [ $? == 0 ]; then 
echo  "fcrepo started..."
else
  echo "can't connect to fcrepo"
  exit 1
fi
date


test_log=$TASK_DIR/test.$SGE_TASK_ID.log

echo "=======" >> $test_log
date >> $test_log
x=1
time ( while [ $x -le $RUN_COUNT ]; do
  host=$(shuf -n 1 $TASK_DIR/hosts)
  echo "POST http://$host/rest/fcr:new"
  curl -X POST "http://$host/rest/fcr:new" &> $TASK_DIR/curl.$SGE_TASK_ID.log

  if [ $? != 0 ]; then 
    echo "Error: $?" >> $test_log
    echo "N: " >> $test_log
    echo $x >> $test_log
    break
  fi

  date >> $test_log
  echo $x >> $test_log

  x=$(( $x + 1 ))
done )
date >> $test_log