NACIS 2016 - Practical Cartography Day

Transcript

1. $ echo “Make, etc.” | \ wall -g PCD Seth

   Fitzsimmons
 Stamen Design, etc. seth@mojodna.net

2. # slides $ open \ http://bit.ly/2eFFvJt

3. # why? # # * repeatable # * self-documenting #

   * transformations # <> data changes

4. # on Windows 10? # no worries # # “Bash

   on Ubuntu # on Windows”

5. # shell fundamentals $ /bin/sh -c theory

6. # is a comment $ is a prompt

7. # <cmd> ––help usually # works # man <cmd> is

   detailed

8. # do this, then that $ this; that

9. # if this, then that $ this && that

10. # that, unless this $ this || that

11. # standard file # descriptors # (stdio)

12. # stdin (fd 0) $ cat > greeting Hi! ⌃d

    $ cat greeting Hi!

13. # stdout (fd 1) $ echo “Hi PCD!”

14. # stderr $ >&2 echo Error

15. # a data black hole /dev/null

16. # redirection $ echo hi > greeting

17. # append $ echo hey >> greeting

18. # stderr → stdout $ thing 2>&1 logs

19. # pipe # this means everything # can be combined!!!

    $ cat /etc/passwd | \ grep -i calendar

20. # exit codes $ thing; echo $?

21. # success (0) $ thing && echo $? 0

22. # failure (not 0) $ thing || echo $? 1

23. $ make basics

24. # same input (repeatedly), # same output $ make idempotency

    Yes $ make idempotency Yes

25. # target this: touch $@

26. # prerequisite this: that # that exists cp that this

27. $ make -j2 vars fns

28. # target this: # outputs “this” echo $@

29. # all prereqs this: that the_other # “that the_other” echo

    $^

30. # first prereq this: that the_other # “that” echo $<

31. # first prereq this: that the_other # “that” echo $<

32. # nth prereq this: that the_other # “the_other” echo $(word

    2, $^)

33. # catch-all %: echo 42 > $@

34. $ ls /bin

35. # explore

36. # who am I? $ whoami seth

37. # where am I? $ pwd /home/seth

38. # what time is it? $ date Tue Oct 18

    23:58:42 MDT 2016

39. # what’s here? $ ls -lh total 8 -rw-r--r-- 1

    seth wheel …

40. # what’s in that file? $ cat /etc/passwd …

41. # …a page at a time $ less /etc/passwd …

42. # …just the beginning $ head /etc/passwd …

43. # …just the end $ tail /etc/passwd …

44. # what variables are set? $ env HOME=/home/seth …

45. # where’s that file? $ find . -type f -name

    hi <list of files>

46. # where’d that file go? # (full-text search, macOS) $

    mdfind Seattle <list of files>

47. # get help $ man man …

48. # manipulation

49. # create a file $ touch file

50. # copy a file $ cp file file2

51. # move a file $ mv file2 file3

52. # delete a file $ rm file3

53. # create a directory $ mkdir -p my/stuff

54. # remove a directory $ rmdir my/stuff

55. # remove a directory # and everything in it $

    rm -r my

56. # find lines in a file $ grep -i name

    file.txt

57. # find non-matching lines $ grep -v name file.txt

58. # count words, lines, # characters $ wc file.txt

59. # pretty-print JSON $ jq . file.json

60. # extract fields $ jq .name file.json

61. # replace things $ sed 's/this/that/' file

62. # extract columns $ cut -d , -f 1,3 file.csv

63. # extract columns $ cut -d , -f 1,3 file.csv

64. # compression

65. # open a zip file $ unzip file.zip

66. # list zip contents $ unzip -v file.zip

67. # create a zip $ zip file file.zip

68. # open a tarball $ tar zxf file.tar.gz

69. # list a tarball $ tar ztf file.tar.gz

70. # create a tarball $ tar zcf file.tar.gz stuff/

71. # compress with gzip $ gzip file.tar

72. # uncompress with gzip $ gzip -d file.tar.gz

73. # misc

74. # always exit 0 $ true; echo $? 0

75. # always exit non-0 $ false; echo $? 1

76. # fetch and fail # if appropriate $ curl -f

    nacis.org

77. # download $ wget nacis.org

78. # download $ wget nacis.org

79. # “open” (macOS) $ open nacis.org

80. # display progress $ cat /etc/passwd | \
 pv |

    wc -l

81. # also write to a file $ echo hi |

    tee file

82. $ man bash

83. #!/usr/bin/env bash

84. set -eo pipefail

85. set -x

86. NACIS=2016

87. # assignment NACIS=“2016”

88. # capture a command NACIS=$(curl nacis.org)

89. echo $NACIS

90. echo ${NACIS}

91. # set a default value echo ${NACIS:-PCD}

92. # replace echo ${NACIS/2016/2017}

93. # replace all echo ${NACIS//2016/2017}

94. # substring echo ${NACIS:2:2}

95. # remove suffix $ filename=“world.tif” $ echo ${filename%.tif} world

96. # do math $ echo $[2 ** 3] 8

97. if [[ “this” != “that” ]]; then echo Control Flow

    elif [[ ! -f file ]]; then touch file elif [[ $six -le $five ]]; then false else rm -f file fi

98. while true; do echo Control Flow done

99. for f in $(ls); do echo $f done

100. # man test

101. # misc

102. # extract filename $ basename /etc/passwd passwd

103. # extract directory $ dirname /etc/passwd /etc

104. $ make recipes

105. $ make convert source.json: source.shp ogr2ogr \ -t_srs EPSG:4326 \

     -f GeoJSON \ $@ \ $<

106. $ make reproject output.tif: source.tif gdalwarp \ -q \ -t_srs

     EPSG:3857 \ $< \ $@

107. $ make wilderness data/S_USA.Wilderness.zip: @mkdir -p $$(dirname $@) @curl -sfL

     http://data.fs.usda.gov/ geodata/edw/edw_resources/shp/ S_USA.Wilderness.zip \ -o $@

108. $ make table db/wilderness: sql/wilderness.sql psql \ -c "\d $(subst

     db/,,$@)" \ > /dev/null \ 2>&1 || \ psql \ -v ON_ERROR_STOP=1 \ -qX1f \ $<

109. # resources • https://bost.ocks.org/mike/make/ • http://www.gregreda.com/2013/07/15/unix- commands-for-data-science/ • https://google.github.io/styleguide/shell.xml •

     https://github.com/stamen/toner-carto/blob/ master/Makefile (WARNING!) • http://mojodna.net/2015/01/07/make-for-data- using-make.html