Allows matterport's MRCNN to run in sections so that every few epochs it re-executes # and, therefore, resets the memory solving provisionally the memory leak issue.

mrcnn_continous_run.sh

# -----------------------------
#   USAGE
# -----------------------------
# chmod +x mrcnn_continous_run.sh
# source mrcnn_continous_run.sh 
#
# DESCRIPTION:
# Allows matterport's MRCNN to run in sections so that every few epochs it re-executes
# and, therefore, resets the memory solving provisionally the memory leak issue. 
#
# Author: Bryan Laygond
# Website: http://www.laygond.com
#
# NOTE:
# - This shell file is meant to run at the same level directory as your python script.
# - This does not directly solve the issue but goes around the problem and therefore not
#   responsible for the execution of this code.
# - You must edit the USER CONFIGURATION section to your needs
# ------------------------------

# USER CONFIGURATION
# Fill in variables (with no space)
exec_cmd='python testingfile.py train --dataset=/path/to/dataset --weights=last'
file='testingfile.py'
line='183'       # line number where the epoch field is indicated in script (i.e.g epochs=300,)
epoch_reset='6'  # reset memory every 6 epochs 

# Get last and final epoch
#   last : the last epoch as indicated in the logs directory
#   final: the final epoch as indicated in your python script
last_epoch=$(ls ../../logs/$(ls ../../logs | tail -n 1) | tail -n 1 | rev | cut -d'.' -f2 | cut -d'_' -f1 | rev | bc)
final_epoch=$(sed "${line}q;d" $file | cut -d'=' -f2 | cut -d',' -f1 )

# Run MRCNN in sections
counter=$last_epoch
while [ $counter -lt $final_epoch ]
do
    counter=$(( counter + epoch_reset ))
    counter=$(( counter < final_epoch ? counter : final_epoch ))
    sed -i "$line c\ \t\t\t\tepochs=$counter," $file
    $exec_cmd
    sleep 10
done