Sweet Unix commands

Unix & Shell Commands

# file copying & filesystem stuff ##############################

# copy massive directories
sudo cp -R -n -p /Volumes/RAID01/_backup_laptop /Volumes/RAID2TB/machine_backups/

# or use rsync
sudo rsync -rltgoDv /Volumes/RAID02/ /Volumes/RAID06/

# diff directories
diff -rq /Volumes/RAID01/_backup_laptop /Volumes/RAID2TB/machine_backups/_backup_laptop/

# copy all files with filenames that match a pattern
cp `find *comp.png` comp/

# copy all files that contain "testtext"
cp `grep -l get *testtext` dir/

# search for huge files
find ./ -size +1000000000c -exec ls -la {} \; 

# delete a folder with stuff in it
rm -rf snowflake-08/

# create a symbolic link (use absolute paths for certainty)
ln -s /Users/cacheflowe/Documents/workspace/bnimbl/bnimbl-web/ /Users/cacheflowe/Documents/workspace/bnimbl/bnimbl-ios/www/bnimbl-web




# git commands ###########################

# set master as origin branch:
git push -u origin master

# bail on all local changes
git checkout -- .

# init git for an existing project:
cd foo_project
git init
git add *
git commit -m "My initial commit message"



# wget and curl scraping #############################

wget -np -nd -r http://spritedatabase.net/objects/1/


curl -o thatpage.html http://www.netscape.com/
curl --trace trace.txt www.haxx.se

curl --trace trace.txt http://feeds.feedburner.com/ffffound/everyone | grep -o "http.*?jpg"


# get all html lines with images, but no height
grep -n 'img' *.php | grep -v 'height'


curl http://feeds.feedburner.com/ffffound/everyone | grep square|cut -d "\"" -f 8|sed 's/\/database/http\:\/\/www.runehq.com\/databasesearch/g'


curl http://www.runehq.com/databasesearch.php | grep square|cut -d "\"" -f 8|sed 's/\/database/http\:\/\/www.runehq.com\/database/g'|sed 's/amp;//g'




wget -r -l 1 -A img http://feeds.feedburner.com/ffffound/everyone
curl http://feeds.feedburner.com/ffffound/everyone > page.txt


wget -r -l 1 -A img http://cacheflowe.com/xml/news.xml

curl http://cacheflowe.com/xml/news.xml > page.txt
grep -o http://[^[:space:]]*.img page > links
wget -i link


curl http://feeds.feedburner.com/ffffound/everyone | grep -o "http.*?jpg"

curl --trace trace.txt http://feeds.feedburner.com/ffffound/everyone | grep -o "http.*?jpg"


curl http://feeds.feedburner.com/ffffound/everyone > page.txt
egrep -o source\ url=\"http://[^[:space:]]*.\(jpg\|png\|gif\) page.txt > links.txt
egrep -o http://[^[:space:]]*.\(jpg\|png\|gif\) links.txt > links_clean.txt
wget -i links_clean.txt



curl http://feeds.feedburner.com/ffffound/everyone | egrep -o source\ url=\"http://[^[:space:]]*.\(jpg\|png\|gif\) | egrep -o http://[^[:space:]]*.\(jpg\|png\|gif\) > links_clean.txt


curl http://feeds.feedburner.com/ffffound/everyone | egrep -o source\ url=\"http://[^[:space:]]*.\(jpg\|png\|gif\) | egrep -o http://[^[:space:]]*.\(jpg\|png\|gif\) | xargs wget


# scrape a whole web site
wget --spider --mirror -p --convert-links -P ./pb http://paperbirdband.com






# ffmpeg commands

# Maybe both of these steps aren't necessary
# ffmpeg -y -loop 1 -i title.png -i silence.m4a -qscale:v 1 -copyts -shortest -t 5 title.mpg
# ffmpeg -y -i title.mpg -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" title.mp4

# Normalizing the user's video
ffmpeg -y -i crap-input.mov -qscale:v 1 -acodec libfaac -ab 96k -vcodec mpeg4 -f mp4 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -copyts -r 30 clip.mp4
ffmpeg -y -i crap-input.mov -vcodec mpeg4 -b:v 10000k -f mp4 clip.mp4
ffmpeg -y -i clip.mp4 -vcodec libx264 -b:v 10000k crap-input-mov.mp4

# Putting the whole thing together
# ffmpeg -y -copyts -f concat -i input.txt -sn -c copy -acodec mp3 -b:a 96k -vcodec libx264 -level 21 -refs 2 -b:v 345k -threads 0 out-ffmpeg.mp4









#!/bin/bash

# Maybe both of these steps aren't necessary
# ffmpeg -y -loop 1 -i title.png -i silence.m4a -acodec libfaac -qscale:v 1 -copyts -shortest -t 5 title.mp4
# ffmpeg -y -i title.mpg -acodec libfaac -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" title.mp4

# # Normalizing the user's video
# ffmpeg -y -i $1 -qscale:v 1 -acodec libfaac -vcodec libx264 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2,transpose=1,transpose=1" -r 30 -f mp4 clip.mp4
# ffmpeg -y -i $1 -acodec libmp3lame -b:a 128k -b:v 3M -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2,transpose=1,transpose=1" -r 30 clip.mpg

# Putting the whole thing together
# ffmpeg -y -copyts -f concat -i input.txt -sn -c copy -acodec mp3 -b:a 96k -vcodec libx264 -level 21 -refs 2 -b:v 345k -threads 0 out-ffmpeg.mp4
ffmpeg -i concat:"doc1.mpg|clip-web-output.mpg|doc2.mpg|doc3.mpg|doc4.mpg|doc5.mpg" -c copy out-ffmpeg.mpg

# convert mpg concat to mp4
# ffmpeg -y -i out-ffmpeg.mpg clip.mp4


# normalize documentary clips
# ffmpeg -y -i inside-out-short-clip-1.mov -qscale:v 1 -acodec libfaac -vcodec libx264 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -copyts -r 30 doc1.mp4
# ffmpeg -y -i inside-out-short-clip-2.mov -qscale:v 1 -acodec libfaac -vcodec libx264 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -copyts -r 30 doc2.mp4
# ffmpeg -y -i inside-out-short-clip-3.mov -qscale:v 1 -acodec libfaac -vcodec libx264 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -copyts -r 30 doc3.mp4
# ffmpeg -y -i inside-out-short-clip-4.mov -qscale:v 1 -acodec libfaac -vcodec libx264 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -copyts -r 30 doc4.mp4
# ffmpeg -y -i inside-out-short-clip-5.mov -qscale:v 1 -acodec libfaac -vcodec libx264 -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -copyts -r 30 doc5.mp4

# normalize documentary clips to mpg
# ffmpeg -y -i inside-out-clip-1.mov -acodec libmp3lame -b:a 128k -b:v 3M -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -r 30 inside-out-clip-1.mpg
# ffmpeg -y -i inside-out-clip-2.mov -acodec libmp3lame -b:a 128k -b:v 3M -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -r 30 inside-out-clip-2.mpg
# ffmpeg -y -i inside-out-clip-3.mov -acodec libmp3lame -b:a 128k -b:v 3M -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -r 30 inside-out-clip-3.mpg
# ffmpeg -y -i inside-out-clip-4.mov -acodec libmp3lame -b:a 128k -b:v 3M -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -r 30 inside-out-clip-4.mpg
# ffmpeg -y -i inside-out-clip-5.mov -acodec libmp3lame -b:a 128k -b:v 3M -vf "scale=iw*min(1280/iw\,720/ih):ih*min(1280/iw\,720/ih),pad=1280:720:(1280-iw)/2:(720-ih)/2" -r 30 inside-out-clip-5.mpg



# Testing MP4box
# MP4Box -add doc1.mp4 -cat clip.mp4 -cat doc2.mp4 -cat doc3.mp4 -cat doc4.mp4 -cat doc5.mp4 -new out-mp4box.mp4

# -vcodec libx264 -b 512k -flags +loop+mv4 -cmp 256 -partitions +parti4x4+parti8x8+partp4x4+partp8x8+partb8x8 -me_method hex -subq 7 -trellis 1 -refs 5 -bf 3 -flags2 +bpyramid+wpred+mixed_refs+dct8x8 -coder 1 -me_range 16 -g 250 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -qmin 10-qmax 51 -qdiff 4