[DRUPAL] Shell script to run a migration in chunks of 3k

migrate-one.sh

ALIAS=@foo
MIGRATION=foo_news
LOG_FILE=/tmp/foo-migrate-logs/migration-foo_news.log

rm -f $LOG_FILE
date > $LOG_FILE

# Run migrations in chunks of 3000 items.
THRESHOLD=3000
TOTAL=$(COLUMNS=160 drush $ALIAS migrate-status $MIGRATION | tail -n 1 | awk '{print $3}')
if [ $TOTAL -gt $THRESHOLD ]; then
  START=1
  while [ ! $START -gt $TOTAL ]; do
    let END="$START + $THRESHOLD - 1"
    echo "### migrate [$START,$END] items ###" >> $LOG_FILE
    drush $ALIAS -d -y migrate-import $MIGRATION --limit=$THRESHOLD >> $LOG_FILE 2>&1
    echo "### migrate-messages ###" >> $LOG_FILE
    drush $ALIAS -y migrate-messages $MIGRATION  >> $LOG_FILE 2>&1
    let START="$END + 1"
  done
fi

# Do a last pass for all items to ensure no one is left over.
echo "### migrate items (last pass) ###" >> $LOG_FILE
drush $ALIAS -d -y migrate-import $MIGRATION >> $LOG_FILE 2>&1
echo "### migrate-messages ###" >> $LOG_FILE
drush $ALIAS -y migrate-messages $MIGRATION  >> $LOG_FILE 2>&1

echo "### migrate-status ###" >> $LOG_FILE
drush $ALIAS migrate-status $MIGRATION >> $LOG_FILE 2>&1
date >> $LOG_FILE

For a 6100 items migration it runs 4 times:

drush mi --limit=3000
drush mi --limit=6000
drush mi --limit=6100
drush mi # no limits, to catch any left over or new item in the source