Pro Yearly is on sale from $80 to $50! »

NACIS 2016 - Practical Cartography Day

NACIS 2016 - Practical Cartography Day

Bash and Make tips for scripting reproducible workflows

A0b49382b6dcd9dd4602c3088717724e?s=128

Seth Fitzsimmons

October 19, 2016
Tweet

Transcript

  1. $ echo “Make, etc.” | \ wall -g PCD Seth

    Fitzsimmons
 Stamen Design, etc. seth@mojodna.net
  2. # slides $ open \ http://bit.ly/2eFFvJt

  3. # why? # # * repeatable # * self-documenting #

    * transformations # <> data changes
  4. # on Windows 10? # no worries # # “Bash

    on Ubuntu # on Windows”
  5. # shell fundamentals $ /bin/sh -c theory

  6. # is a comment $ is a prompt

  7. # <cmd> ––help usually # works # man <cmd> is

    detailed
  8. # do this, then that $ this; that

  9. # if this, then that $ this && that

  10. # that, unless this $ this || that

  11. # standard file # descriptors # (stdio)

  12. # stdin (fd 0) $ cat > greeting Hi! ⌃d

    $ cat greeting Hi!
  13. # stdout (fd 1) $ echo “Hi PCD!”

  14. # stderr $ >&2 echo Error

  15. # a data black hole /dev/null

  16. # redirection $ echo hi > greeting

  17. # append $ echo hey >> greeting

  18. # stderr → stdout $ thing 2>&1 logs

  19. # pipe # this means everything # can be combined!!!

    $ cat /etc/passwd | \ grep -i calendar
  20. # exit codes $ thing; echo $?

  21. # success (0) $ thing && echo $? 0

  22. # failure (not 0) $ thing || echo $? 1

  23. $ make basics

  24. # same input (repeatedly), # same output $ make idempotency

    Yes $ make idempotency Yes
  25. # target this: touch $@

  26. # prerequisite this: that # that exists cp that this

  27. $ make -j2 vars fns

  28. # target this: # outputs “this” echo $@

  29. # all prereqs this: that the_other # “that the_other” echo

    $^
  30. # first prereq this: that the_other # “that” echo $<

  31. # first prereq this: that the_other # “that” echo $<

  32. # nth prereq this: that the_other # “the_other” echo $(word

    2, $^)
  33. # catch-all %: echo 42 > $@

  34. $ ls /bin

  35. # explore

  36. # who am I? $ whoami seth

  37. # where am I? $ pwd /home/seth

  38. # what time is it? $ date Tue Oct 18

    23:58:42 MDT 2016
  39. # what’s here? $ ls -lh total 8 -rw-r--r-- 1

    seth wheel …
  40. # what’s in that file? $ cat /etc/passwd …

  41. # …a page at a time $ less /etc/passwd …

  42. # …just the beginning $ head /etc/passwd …

  43. # …just the end $ tail /etc/passwd …

  44. # what variables are set? $ env HOME=/home/seth …

  45. # where’s that file? $ find . -type f -name

    hi <list of files>
  46. # where’d that file go? # (full-text search, macOS) $

    mdfind Seattle <list of files>
  47. # get help $ man man …

  48. # manipulation

  49. # create a file $ touch file

  50. # copy a file $ cp file file2

  51. # move a file $ mv file2 file3

  52. # delete a file $ rm file3

  53. # create a directory $ mkdir -p my/stuff

  54. # remove a directory $ rmdir my/stuff

  55. # remove a directory # and everything in it $

    rm -r my
  56. # find lines in a file $ grep -i name

    file.txt
  57. # find non-matching lines $ grep -v name file.txt

  58. # count words, lines, # characters $ wc file.txt

  59. # pretty-print JSON $ jq . file.json

  60. # extract fields $ jq .name file.json

  61. # replace things $ sed 's/this/that/' file

  62. # extract columns $ cut -d , -f 1,3 file.csv

  63. # extract columns $ cut -d , -f 1,3 file.csv

  64. # compression

  65. # open a zip file $ unzip file.zip

  66. # list zip contents $ unzip -v file.zip

  67. # create a zip $ zip file file.zip

  68. # open a tarball $ tar zxf file.tar.gz

  69. # list a tarball $ tar ztf file.tar.gz

  70. # create a tarball $ tar zcf file.tar.gz stuff/

  71. # compress with gzip $ gzip file.tar

  72. # uncompress with gzip $ gzip -d file.tar.gz

  73. # misc

  74. # always exit 0 $ true; echo $? 0

  75. # always exit non-0 $ false; echo $? 1

  76. # fetch and fail # if appropriate $ curl -f

    nacis.org
  77. # download $ wget nacis.org

  78. # download $ wget nacis.org

  79. # “open” (macOS) $ open nacis.org

  80. # display progress $ cat /etc/passwd | \
 pv |

    wc -l
  81. # also write to a file $ echo hi |

    tee file
  82. $ man bash

  83. #!/usr/bin/env bash

  84. set -eo pipefail

  85. set -x

  86. NACIS=2016

  87. # assignment NACIS=“2016”

  88. # capture a command NACIS=$(curl nacis.org)

  89. echo $NACIS

  90. echo ${NACIS}

  91. # set a default value echo ${NACIS:-PCD}

  92. # replace echo ${NACIS/2016/2017}

  93. # replace all echo ${NACIS//2016/2017}

  94. # substring echo ${NACIS:2:2}

  95. # remove suffix $ filename=“world.tif” $ echo ${filename%.tif} world

  96. # do math $ echo $[2 ** 3] 8

  97. if [[ “this” != “that” ]]; then echo Control Flow

    elif [[ ! -f file ]]; then touch file elif [[ $six -le $five ]]; then false else rm -f file fi
  98. while true; do echo Control Flow done

  99. for f in $(ls); do echo $f done

  100. # man test

  101. # misc

  102. # extract filename $ basename /etc/passwd passwd

  103. # extract directory $ dirname /etc/passwd /etc

  104. $ make recipes

  105. $ make convert source.json: source.shp ogr2ogr \ -t_srs EPSG:4326 \

    -f GeoJSON \ $@ \ $<
  106. $ make reproject output.tif: source.tif gdalwarp \ -q \ -t_srs

    EPSG:3857 \ $< \ $@
  107. $ make wilderness data/S_USA.Wilderness.zip: @mkdir -p $$(dirname $@) @curl -sfL

    http://data.fs.usda.gov/ geodata/edw/edw_resources/shp/ S_USA.Wilderness.zip \ -o $@
  108. $ make table db/wilderness: sql/wilderness.sql psql \ -c "\d $(subst

    db/,,$@)" \ > /dev/null \ 2>&1 || \ psql \ -v ON_ERROR_STOP=1 \ -qX1f \ $<
  109. # resources • https://bost.ocks.org/mike/make/ • http://www.gregreda.com/2013/07/15/unix- commands-for-data-science/ • https://google.github.io/styleguide/shell.xml •

    https://github.com/stamen/toner-carto/blob/ master/Makefile (WARNING!) • http://mojodna.net/2015/01/07/make-for-data- using-make.html