Skip to content

Instantly share code, notes, and snippets.

@pudo

pudo/deployscraper.sh

Last active February 4, 2016 14:52
Show Gist options
  • Save pudo/124e41b66fec6b592a47 to your computer and use it in GitHub Desktop.
Save pudo/124e41b66fec6b592a47 to your computer and use it in GitHub Desktop.
Download ZIP
deployscraper
Raw
deployscraper.sh
apt-get update -qq
apt-get install -y -q tmux vim git python-virtualenv curl python-pip build-essential python-dev libxml2-dev libxslt1-dev libpq-dev apt-utils ca-certificates unrar-free unzip
pip install lxml dataset metafolder requests csvkit psycopg2 thready unicodecsv
mkdir -p /srv/data/source
chmod 0600 ~/.ssh/id_rsa
echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrzARRFK4JzzvNLzKvWMlnlFBO9D9g3mxvr9Cji4K7psaXHMYN8sv5z+7HaNgee3mpXWvDNnkwZRJj0Cv+H2KxSwfOCRD0ubVmGfLh+3PKlso3bq5aHN0DOcK44Z7j/twE9SMumzpIWGSRrdghGu+XaKu90y/NFxLCwUhZ4EmMf+hFup24c8JLFZ4GFVTZCGveTUri7a9g57RTkJPOuJNDrOCTL3dw1ebxhd8TBm3EfeQ0Fj8o7k0PojJ6OcHpV8Evy4h455ioiJfHrrKif3JUlzC0H2eKWGEQHwEYBjYZVl0dktL7pP7iDG/zO7lzMwgCjXAjYZH+I4zFP8L9le93 occrpscraper" >>~/.ssh/authorized_keys
echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrzARRFK4JzzvNLzKvWMlnlFBO9D9g3mxvr9Cji4K7psaXHMYN8sv5z+7HaNgee3mpXWvDNnkwZRJj0Cv+H2KxSwfOCRD0ubVmGfLh+3PKlso3bq5aHN0DOcK44Z7j/twE9SMumzpIWGSRrdghGu+XaKu90y/NFxLCwUhZ4EmMf+hFup24c8JLFZ4GFVTZCGveTUri7a9g57RTkJPOuJNDrOCTL3dw1ebxhd8TBm3EfeQ0Fj8o7k0PojJ6OcHpV8Evy4h455ioiJfHrrKif3JUlzC0H2eKWGEQHwEYBjYZVl0dktL7pP7iDG/zO7lzMwgCjXAjYZH+I4zFP8L9le93 occrpscraper" >~/.ssh/id_rsa.pub
git clone [email protected]:occrp/datavault.git /srv/etl
pip install -r /srv/etl/requirements.txt
pip install -e /srv/etl/lib
echo "export VIRTUAL_ENV=/usr" >/srv/etl/env.sh
echo "export DATAVAULT_DATABASE_URI=sqlite:////srv/db.sqlite3" >>/srv/etl/env.sh
echo "export DATAVAULT_DATA_PATH=/srv/data/source" >>/srv/etl/env.sh
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment