Twitter stream sucker. Drinks from the Gardenhose.

stream_sucker.sh

#!/bin/sh

#
# A Twitter stream sucker. Writes one file per hour, containing all the tweets from the
# sample stream (aka "gardenhose").
#
# Use Ctrl-C to stop collecting.
#
# Note: This is not intended to be a robust tool for collecting Twitter data. Use at your 
# own risk, and play nice: http://dev.twitter.com/pages/streaming_api_concepts
#
# Note 2: This script is likely to cease working when Twitter drops support for basic
# authentication for stream connections. You have been warned.
#

# fill in these:
USER=YOUR_USERNAME
PASS=YOUR_PASSWORD

SECS_PER_FILE=3600

OUTPUT_DIR=$1
echo "Writing output to $OUTPUT_DIR"

while true; do
  OUTPUT_FILE=$OUTPUT_DIR/$( date +%Y%m%d_%H%M%S ).json
  echo "Creating file $OUTPUT_FILE"
  curl http://stream.twitter.com/1/statuses/sample.json -u${USER}:${PASS} -m $SECS_PER_FILE > $OUTPUT_FILE
  if [[ $? -ne 0 ]]; then
    echo "curl died with an error, stopping collection"
    break
  fi
done