$30 off During Our Annual Pro Sale. View Details »

NACIS 2016 - Practical Cartography Day

NACIS 2016 - Practical Cartography Day

Bash and Make tips for scripting reproducible workflows

Seth Fitzsimmons

October 19, 2016
Tweet

More Decks by Seth Fitzsimmons

Other Decks in Design

Transcript

  1. $ echo “Make, etc.” | \
    wall -g PCD
    Seth Fitzsimmons

    Stamen Design, etc.

    [email protected]

    View Slide

  2. # slides
    $ open \
    http://bit.ly/2eFFvJt

    View Slide

  3. # why?
    #
    # * repeatable
    # * self-documenting
    # * transformations
    # <> data changes

    View Slide

  4. # on Windows 10?
    # no worries
    #
    # “Bash on Ubuntu
    # on Windows”

    View Slide

  5. # shell fundamentals
    $ /bin/sh -c theory

    View Slide

  6. # is a comment
    $ is a prompt

    View Slide

  7. # ––help usually
    # works
    # man is detailed

    View Slide

  8. # do this, then that
    $ this; that

    View Slide

  9. # if this, then that
    $ this && that

    View Slide

  10. # that, unless this
    $ this || that

    View Slide

  11. # standard file
    # descriptors
    # (stdio)

    View Slide

  12. # stdin (fd 0)
    $ cat > greeting
    Hi!
    ⌃d
    $ cat greeting
    Hi!

    View Slide

  13. # stdout (fd 1)
    $ echo “Hi PCD!”

    View Slide

  14. # stderr
    $ >&2 echo Error

    View Slide

  15. # a data black hole
    /dev/null

    View Slide

  16. # redirection
    $ echo hi > greeting

    View Slide

  17. # append
    $ echo hey >> greeting

    View Slide

  18. # stderr → stdout
    $ thing 2>&1 logs

    View Slide

  19. # pipe
    # this means everything
    # can be combined!!!
    $ cat /etc/passwd | \
    grep -i calendar

    View Slide

  20. # exit codes
    $ thing; echo $?

    View Slide

  21. # success (0)
    $ thing && echo $?
    0

    View Slide

  22. # failure (not 0)
    $ thing || echo $?
    1

    View Slide

  23. $ make basics

    View Slide

  24. # same input (repeatedly),
    # same output
    $ make idempotency
    Yes
    $ make idempotency
    Yes

    View Slide

  25. # target
    this:
    touch $@

    View Slide

  26. # prerequisite
    this: that
    # that exists
    cp that this

    View Slide

  27. $ make -j2 vars fns

    View Slide

  28. # target
    this:
    # outputs “this”
    echo $@

    View Slide

  29. # all prereqs
    this: that the_other
    # “that the_other”
    echo $^

    View Slide

  30. # first prereq
    this: that the_other
    # “that”
    echo $<

    View Slide

  31. # first prereq
    this: that the_other
    # “that”
    echo $<

    View Slide

  32. # nth prereq
    this: that the_other
    # “the_other”
    echo $(word 2, $^)

    View Slide

  33. # catch-all
    %:
    echo 42 > $@

    View Slide

  34. $ ls /bin

    View Slide

  35. # explore

    View Slide

  36. # who am I?
    $ whoami
    seth

    View Slide

  37. # where am I?
    $ pwd
    /home/seth

    View Slide

  38. # what time is it?
    $ date
    Tue Oct 18 23:58:42 MDT 2016

    View Slide

  39. # what’s here?
    $ ls -lh
    total 8
    -rw-r--r-- 1 seth wheel …

    View Slide

  40. # what’s in that file?
    $ cat /etc/passwd

    View Slide

  41. # …a page at a time
    $ less /etc/passwd

    View Slide

  42. # …just the beginning
    $ head /etc/passwd

    View Slide

  43. # …just the end
    $ tail /etc/passwd

    View Slide

  44. # what variables are set?
    $ env
    HOME=/home/seth

    View Slide

  45. # where’s that file?
    $ find . -type f -name hi

    View Slide

  46. # where’d that file go?
    # (full-text search, macOS)
    $ mdfind Seattle

    View Slide

  47. # get help
    $ man man

    View Slide

  48. # manipulation

    View Slide

  49. # create a file
    $ touch file

    View Slide

  50. # copy a file
    $ cp file file2

    View Slide

  51. # move a file
    $ mv file2 file3

    View Slide

  52. # delete a file
    $ rm file3

    View Slide

  53. # create a directory
    $ mkdir -p my/stuff

    View Slide

  54. # remove a directory
    $ rmdir my/stuff

    View Slide

  55. # remove a directory
    # and everything in it
    $ rm -r my

    View Slide

  56. # find lines in a file
    $ grep -i name file.txt

    View Slide

  57. # find non-matching lines
    $ grep -v name file.txt

    View Slide

  58. # count words, lines,
    # characters
    $ wc file.txt

    View Slide

  59. # pretty-print JSON
    $ jq . file.json

    View Slide

  60. # extract fields
    $ jq .name file.json

    View Slide

  61. # replace things
    $ sed 's/this/that/' file

    View Slide

  62. # extract columns
    $ cut -d , -f 1,3 file.csv

    View Slide

  63. # extract columns
    $ cut -d , -f 1,3 file.csv

    View Slide

  64. # compression

    View Slide

  65. # open a zip file
    $ unzip file.zip

    View Slide

  66. # list zip contents
    $ unzip -v file.zip

    View Slide

  67. # create a zip
    $ zip file file.zip

    View Slide

  68. # open a tarball
    $ tar zxf file.tar.gz

    View Slide

  69. # list a tarball
    $ tar ztf file.tar.gz

    View Slide

  70. # create a tarball
    $ tar zcf file.tar.gz stuff/

    View Slide

  71. # compress with gzip
    $ gzip file.tar

    View Slide

  72. # uncompress with gzip
    $ gzip -d file.tar.gz

    View Slide

  73. # misc

    View Slide

  74. # always exit 0
    $ true; echo $?
    0

    View Slide

  75. # always exit non-0
    $ false; echo $?
    1

    View Slide

  76. # fetch and fail
    # if appropriate
    $ curl -f nacis.org

    View Slide

  77. # download
    $ wget nacis.org

    View Slide

  78. # download
    $ wget nacis.org

    View Slide

  79. # “open” (macOS)
    $ open nacis.org

    View Slide

  80. # display progress
    $ cat /etc/passwd | \

    pv | wc -l

    View Slide

  81. # also write to a file
    $ echo hi | tee file

    View Slide

  82. $ man bash

    View Slide

  83. #!/usr/bin/env bash

    View Slide

  84. set -eo pipefail

    View Slide

  85. set -x

    View Slide

  86. NACIS=2016

    View Slide

  87. # assignment
    NACIS=“2016”

    View Slide

  88. # capture a command
    NACIS=$(curl nacis.org)

    View Slide

  89. echo $NACIS

    View Slide

  90. echo ${NACIS}

    View Slide

  91. # set a default value
    echo ${NACIS:-PCD}

    View Slide

  92. # replace
    echo ${NACIS/2016/2017}

    View Slide

  93. # replace all
    echo ${NACIS//2016/2017}

    View Slide

  94. # substring
    echo ${NACIS:2:2}

    View Slide

  95. # remove suffix
    $ filename=“world.tif”
    $ echo ${filename%.tif}
    world

    View Slide

  96. # do math
    $ echo $[2 ** 3]
    8

    View Slide

  97. if [[ “this” != “that” ]]; then
    echo Control Flow
    elif [[ ! -f file ]]; then
    touch file
    elif [[ $six -le $five ]]; then
    false
    else
    rm -f file
    fi

    View Slide

  98. while true; do
    echo Control Flow
    done

    View Slide

  99. for f in $(ls); do
    echo $f
    done

    View Slide

  100. # man test

    View Slide

  101. # misc

    View Slide

  102. # extract filename
    $ basename /etc/passwd
    passwd

    View Slide

  103. # extract directory
    $ dirname /etc/passwd
    /etc

    View Slide

  104. $ make recipes

    View Slide

  105. $ make convert
    source.json: source.shp
    ogr2ogr \
    -t_srs EPSG:4326 \
    -f GeoJSON \
    $@ \
    $<

    View Slide

  106. $ make reproject
    output.tif: source.tif
    gdalwarp \
    -q \
    -t_srs EPSG:3857 \
    $< \
    $@

    View Slide

  107. $ make wilderness
    data/S_USA.Wilderness.zip:
    @mkdir -p $$(dirname $@)
    @curl -sfL http://data.fs.usda.gov/
    geodata/edw/edw_resources/shp/
    S_USA.Wilderness.zip \
    -o $@

    View Slide

  108. $ make table
    db/wilderness: sql/wilderness.sql
    psql \
    -c "\d $(subst db/,,$@)" \
    > /dev/null \
    2>&1 || \
    psql \
    -v ON_ERROR_STOP=1 \
    -qX1f \
    $<

    View Slide

  109. # resources
    • https://bost.ocks.org/mike/make/
    • http://www.gregreda.com/2013/07/15/unix-
    commands-for-data-science/
    • https://google.github.io/styleguide/shell.xml
    • https://github.com/stamen/toner-carto/blob/
    master/Makefile (WARNING!)
    • http://mojodna.net/2015/01/07/make-for-data-
    using-make.html

    View Slide