My magic makefile to make everything happen

Makefile

SPACY_MODEL ?= xx_ent_wiki_sm
PYTHONPATH ?= ~/django_home
DJANGO_SETTINGS_MODULE ?= django_home.settings

%.min.js: %.js
	uglifyjs < $< > $@

%.xml: %.edn
	edn2xml < $< > $@

%.bundle.js: %.js
	browserify $< > $@

%.tab: %.mysql.sql
	mysql --defaults-file=my.cnf < $< > $@

%.xml: %.mysql.sql
	mysql --defaults-file=my.cnf --xml < $< > $@

%.html: %.psql.sql
	psql --html < $< > $@

%.xml: %.psql.sql
	# TODO fix double quotes
	echo "select concat('<?xml version=\"1.0\"?>',query_to_xml(:q, false, false, ''));" \
	    | psql -t -A -v q="'$(shell cat $<)'" \
	    > $@


%.xml: %.html
	xsltproc --html $*.xsl $< -o $@

%.txt: %.pdf
	pdftotext - - < $< > $@

%.info.txt: %.pdf
	pdfinfo $< > $@

%.pdf: %.svg
	cairosvg $< -o $@

%.out: %.curl
	curl -K $< > $@

%.models.py:
	PYTHONPATH=$(PYTHONPATH) DJANGO_SETTINGS_MODULE=$(DJANGO_SETTINGS_MODULE) python3 -m django inspectdb $(shell basename $*) > $@

U.S.DividendChampions.xlsx:
	curl -L https://bitly.com/USDividendChampions > $@

%.dividends.csv:
	curl "https://query1.finance.yahoo.com/v7/finance/download/$(shell basename $*)?period1=0&period2=$(shell date +%s)&interval=1d&events=div" > $@

%.sorted.csv: %.csv
	csvsort < $< > $@

%/stocks.nyse.min.json:
	curl 'https://www.nyse.com/api/quotes/filter' \
	    -XPOST \
	    -H 'Content-Type: application/json' \
	    --data-binary '{"instrumentType":"EQUITY","pageNumber":1,"maxResultsPerPage":100000}' > $@

%/dividends: %/stock_symbols.nyse.csv.tail
	mkdir -p $@
	cat $< | xargs printf "$@/%s.dividends.sorted.csv\n" | xargs $(MAKE)

%/stock_symbols.nyse.csv: %/stocks.nyse.csv
	cat $< | csvcut -c symbolTicker > $@

%.tail: %
	tail -n +2 < $< > $@

%.csv: %.json
	json2csv < $< > $@

%.json: %.iob
	python3 -m spacy convert $< > $@

%.table.txt: %.csv
	csvlook $@ > $@

%.json: %.min.json
	jq < $< > $@

%.min.json: %.csv
	csvjson < $< > $@

%.out.edn: %.edn

%.xml: %.php
	php < $< > $@

%.svg: %.csv
	gnuplot -e "set key autotitle columnhead; set datafile separator ','; set terminal svg enhanced mouse jsdir 'http://gnuplot.sourceforge.net/demo_svg_5.2'; plot '$<';" > $@

%.instagram.jpg:
	curl -L "https://www.instagram.com/p/$(shell basename $*)/media?size=l" > $@

# Sign in doesn't work
music/%:
	@mkdir -p `dirname $@`
	printf "<iframe \
	    src='https://embed.music.apple.com/us/album/$(shell basename $*)' \
	    height='100%%' \
	    frameborder='0' \
	    allow='autoplay *; encrypted-media *;' \
	    sandbox='allow-forms allow-popups allow-same-origin allow-scripts allow-storage-access-by-user-activation allow-top-navigation-by-user-activation' \
	    width='100%%' \
	    ></iframe>" > $@

youtube/%:
	@mkdir -p `dirname $@`
	printf "<iframe \
	    src='https://www.youtube.com/embed/$(shell basename $*)' \
	    height='100%%' \
	    frameborder='0' \
	    width='100%%' \
	    ></iframe>" > $@

yad2/rentals/%.json:
	@mkdir -p `dirname $@`
	@res="$(shell curl 'https://www.yad2.co.il/api/pre-load/getFeedIndex/realestate/rent?page=$*' \
	    -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/$*.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15' \
	    -f \
	    -L \
	    -s \
	    -w %{url_effective} \
	    -o $@)" \
	    ; file --mime $@ | grep "text/html" \
	    && echo "Error: Verification required" \
	    && rm $@ && open $$res\
	    && exit 1 \
	    || exit 0

domains/%:
	mkdir -p $(shell dirname $@)
	whois $* > $@
	nslookup $* >> $@

%.svg: %.geojson
	geo2svg < $< > $@

%.kml: %.geojson
	./node_modules/.bin/tokml $< > $@

server:
	shell2http -cgi / 'export FILE=.`echo $$REQUEST_URI | sed  "s/?.*//"` && make -s $$FILE && cat $$FILE'

corona.geojson:
	curl "https://services5.arcgis.com/dlrDjz89gx9qyfev/arcgis/rest/services/Corona_Exposure_View/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token=" -o $@

%.leaflet.json: %.geojson
	jq -s ".[0].geoJSON=.[1] | first" leaflet.json $< > $@

%.json: %.xml
	xml2json < $< > $@

%.json.js: %.json
	browserify --require ./$< -s $(shell basename $*) > $@

%.json.xml: %.json
	echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"leaflet.xsl\" type=\"text/xsl\"?><root><![CDATA[$$(cat $<)]]></root>" > $@

%.leaflet.html: $.leaflet.json

%.sql: %.csv
	csvsql $< > $@

airbnb/%:
	curl "https://www.airbnb.com/api/v2/pdp_listing_booking_details?_format=for_web_with_date&check_in=2020-03-29&check_out=2020-04-21&currency=ILS&guests=1&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&listing_id=$*&locale=en&number_of_adults=1&number_of_children=0&number_of_infants=0" \
	    | jq ".pdp_listing_booking_details[0].price.total.amount" \
	    > $@

%.spacy.json: %.txt
	python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("$(SPACY_MODEL)")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@

%.ent.spacy.html: %.txt
	python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("$(SPACY_MODEL)")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@

%.png: %.pdf
	cd $(shell dirname $@) && pdftocairo -png -singlefile $(shell basename $<)

dividend-aristocrats.csv:
	curl -L http://www.simplysafedividends.com/intelligent-income/idea_lists/3-dividend-aristocrats.csv > $@

%.test.iob: %.iob
	cat $< | head -n $(shell echo $(shell cat $< | wc -l) \* 0.6 / 1 | bc) > $@

%.dev.iob: %.iob
	cat $< | head -n $(shell echo $(shell cat $< | wc -l) \* 0.4 / 1 | bc) > $@

%.model: %.test.json %.dev.json
	rm -rf $@
	python3 -m spacy train \
	    -p ner \
	    -b xx_ent_wiki_sm\
	    xx $@ $*.test.json $*.dev.json

clean:
	git clean -fdX

.PHONY: %/dividends *.php clean