Makefile

#Similar Makefile which this format was inspired by: https://github.com/scities-data/metro-atlas_2014/blob/master/Makefile
# To a lesser extent, this guide https://github.com/datamade/data-making-guidelines and
# also Mike Bostock's article: https://bost.ocks.org/mike/make/

# := assignment only gets run once at the beginning of everything
# = assignment gets run each time (but can cause an infinite loop)

.DEFAULT_GOAL := all

all: data/geojson/combined_smooth_1pc.geojson

#################
# DOWNLOAD DATA #
#################

#prereqs: p7zip csvkit

data/gz/ne_10m_admin_0_countries.zip:
	mkdir -p $(dir $@)
	curl "http://naciscdn.org/naturalearth/10m/cultural/ne_10m_admin_0_countries.zip" -o $@.download
	mv $@.download $@

#################
# UNCOMPRESS DATA #
#################

data/shp/ne_10m_admin_0_countries.shp: data/gz/ne_10m_admin_0_countries.zip
	rm -rf $(basename $@)
	mkdir -p $(basename $@)
	unzip -d $(basename $@) $<
	for file in $(basename $@)/*; do chmod 644 $$file; mv $$file $(basename $@).$${file##*.}; done
	rmdir $(basename $@)
	touch $@

#######################
# CONVERT RASTER DATA TO VECTOR  #
#######################

DOCKERID= $(shell cat data/docker.pid) #"98aa123a0d43"

data/docker.pid:
	mkdir -p $(dir $@)
	#echo DISPLAY IP address must be your local machine IP, $DISPLAY might work on Linux.
	#on Mac you must run socat as described at http://kartoza.com/en/blog/how-to-run-a-linux-gui-application-on-osx-using-docker/A
	# on Mac, "-e DISPLAY=$(shell ipconfig getifaddr en0):0" (no quotes) in place of the static IP address will retrieve yours dynamically
	# you may want to map your home directory with (no quotes) e.g. "	 -v /Users/thadk:/home/thadk  \" 
	#
	#On DockerHub, thadk/qgis-desktop:2.8.9a is a stable, official old build of kartoza/qgis-desktop:2.8 which has since broken.
	@echo "Docker was freshly started, you may need to reinitiate the process"
	docker run --detach -i -t \
	 -v $$(pwd)/rasterScripts/.config:/root/.config \
	 -v $$(pwd):/opt/data thadk/qgis-desktop:2.8.9a /bin/bash > $@

data/docker.aptgetupdate:
	@echo "Trying to run apt-get for java: Docker was freshly started, you may need to reinitiate the process"
	mkdir -p $(dir $@)
	docker exec -i -t $(DOCKERID) apt-get update
	# imagemagick is used to make images monochrome before polygonize
	docker exec -i -t $(DOCKERID) apt-get install imagemagick -y
	#xvfb is used to have headless WorldFileTool
	docker exec -i -t $(DOCKERID) apt-get install xvfb -y
	#Java is needed for WorldFileTool
	docker exec -i -t $(DOCKERID) apt-get install openjdk-7-jre -y
	touch $@

# A series of rules to generate a list of KML files from data/kml/* folder
#  and then to append them into data/shp/*_attrdissolve.shp.
# also, require docker to be running and updated.
.PHONEY: all-kml-shp
kmllist := $(notdir $(shell ls data/kml/*.kml 2>/dev/null))
kmllist_shp := $(addprefix data/shp/,$(addsuffix _attrdissolve.shp,$(basename ${kmllist})))
all-kml-shp: data/docker.pid data/docker.aptgetupdate  ${kmllist_shp} ; echo "$@ success"

# placeholder file to represent tasks needed to prepare your folder of KML files with GroundOverlay's.
# I believe these KMLs need to be unzipped from KMZ zip format.
data/kml/KmlFolder.placeholder:
	mkdir -p $(dir $@)
	touch $@

#xvfb-run is used to avoid needing a X server to run this jar.
#TODO make it work on a single KML
data/shp/%_basic.shp: data/kml/KmlFolder.placeholder data/docker.pid data/docker.aptgetupdate
	mkdir -p $(dir $@)
	#
	# Generate a standard WorldFile "sidecar" file from the KML using this java tool
	docker exec -i -t $(DOCKERID) xvfb-run java -jar \
	 /opt/data/rasterScripts/WorldFileTool.jar \
	 /opt/data/$(dir $<)$*.kml
	#
	# Convert to monochrome so polygonize can easily find the lines to trace
	docker exec -i -t $(DOCKERID) convert \
	 -level 100%,100%,100%  /opt/data/$(dir $<)$*.png -level 100%,100%,100% /opt/data/$(dir $<)$*-mono.png
	cp $(dir $<)$*.pgw $(dir $<)$*-mono.pgw
	#
	# then run GDAL's polygonize feature.
	docker exec -i -t $(DOCKERID) gdal_polygonize.py \
	 /opt/data/$(dir $<)$*-mono.png -f "ESRI Shapefile" /opt/data/$@

#######################
# RESHAPE VECTOR DATA  #
#######################

# The GRASS process below cleans the polygons generated by polygonize and mapshaper's simplify function
# so they can be used in other vector operations like unions or intersects.
# it is possible that PostGIS repair or QGIS "Fix Geometries" features could be used instead, if available.
# these will load GRASS using Docker.

.SECONDARY:

# You may need to provide an appropriate prj file as the worldfiletool will not generate one.
# Uses: geodata/grass /data/data/grassdb/here/PERMANENT
# Applies a "snakes" GRASS simplification on the perimeter to make it look friendlier.
data/shp/%_smooth.shp: data/shp/%_basic.shp
	mkdir -p $(dir $@)
	cp vectorScripts/default.prj $(dir $@)$(notdir $*)_basic.prj
	docker run -it --rm -e TARGET_GRASS_OUT=$@ -e TARGET_GRASS_SHP=$(dir $@)$(notdir $*)_basic.shp \
	 -e GRASS_GUI="text" -e GRASS_BATCH_JOB='/data/vectorScripts/grassGeneralizeSnakes.sh' \
	 -v $$(pwd):/data \
	 geodata/grass -c /data/grassdb/here
	sudo rm -rf ./grassdb

data/shp/%_smooth_small_dirty.shp: data/shp/%_smooth.shp
	#simply the polygons (will be low quality/messy, not union-able)
	mapshaper -i $< \
	 -simplify "1.1%" weighted stats \
	 -o format=shapefile $@

data/shp/%_smooth_small.shp: data/shp/%_smooth_small_dirty.shp
	docker run -it --rm -e TARGET_GRASS_OUT=$@ -e TARGET_GRASS_SHP=$(dir $@)$(notdir $*)_smooth_small_dirty.shp \
	 -e GRASS_GUI="text" -e GRASS_BATCH_JOB='/data/vectorScripts/grassSimpleClean.sh' \
	 -v $$(pwd):/data \
	 geodata/grass -c /data/grassdb/here
	sudo rm -rf ./grassdb

data/shp/%_attr.shp: data/shp/%_smooth_small.shp 
	# reshape your file as needed (join with other geodata perhaps)
	mapshaper -i $< \
	 -o format=shapefile $@

data/shp/%_attrdissolve.shp: data/shp/%_smooth_small.shp
	#optionally dissolve the polygons into one multipolygon (then not filterable later) using -dissolve,
	# then join in the attributes from our master file.
	mapshaper -i $< \
	 -dissolve \
	 -o format=shapefile $@

#this command tells make to retain intermediate files between the first rule and the rule you requested
.PRECIOUS: data/shp/%.shp

#summarize the output into geojson
data/geojson/combined_smooth_1pc.geojson: all-kml-shp
	mkdir -p $(dir $@)
	mapshaper -i data/shp/*_attrdissolve.shp combine-files \
	 -merge-layers \
	 -o format=geojson $@

#====
#Cleaning
#====

clean: DATESHORT:=$(shell date +%Y-%m-%d-%H-%M-%S)
clean: JUNKBIN=~~/Downloads/VMs
clean:
	mkdir -p ${JUNKBIN}
	-docker stop $(DOCKERID)
	-docker rm $(DOCKERID)
	-rm -rf data/docker.pid data/docker.aptgetupdate
	-mv data/shp ${JUNKBIN}/shp-$(DATESHORT)
	-mv data/geojson ${JUNKBIN}/geojson-$(DATESHORT)
	-mv data/csv ${JUNKBIN}/csv-$(DATESHORT)

clean-docker:
	rm -rf data/docker.pid data/docker.aptgetupdate