Data wrangling & manipulation in R - Day 1 slides

Transcript

1. data_wrangling() && ("manipulation" %in% R) postgraduate_workshop( dept = "Biological Sciences", presenter = c( "Ruan van Mazijk", "MSc candidate" ) ) %>% %>% %>% > logos() > face()
2. > introduce( )
3. > introduce( ) • BSc + Hons here at UCT
4. > introduce( ) • BSc + Hons here at UCT • Ecology & evolution • (Mostly plant) comparative biology • Biogeography
5. > introduce( ) • BSc + Hons here at UCT • Ecology & evolution • (Mostly plant) comparative biology • Biogeography • Been working with R for 4½ years • Every major project I’ve done…
6. > introduce( ) Schoenus compar Silvermine, Table Mountatin NP R. van Mazijk 2018 Tetraria ustulata Marloth NR R. van Mazijk 2018 Tetraria thermalis Silvermine, Table Mountain NP R. van Mazijk 2018
7. > workshop$goals
8. > workshop$goals • More reproducible science
9. > workshop$goals • More reproducible science • Save time by: • Automating repetitive tasks • Eliminating human error
10. > workshop$goals • More reproducible science • Save time by: • Automating repetitive tasks • Eliminating human error • Boost your skills • Think about your data programmatically
11. tinyurl.com/r-with-ruan Notes & slides will go up here: (But I encourage you to make your own notes!)
12. > workshop$outline
13. > workshop$outline[1:3]
14. > workshop$outline[1:3] DAY 1 Tidy data principles & tidyr
15. > workshop$outline[1:3] DAY 1 Tidy data principles & tidyr DAY 2 Manipulating data & an intro to dplyr DAY 3 Extending your data with mutate(), summarise() & friends
16. > workshop$outline[-(1:3)]
17. > workshop$outline[-(1:3)] 2 dialects of R:
18. > workshop$outline[-(1:3)] 2 dialects of R: base $ [] [[]] apply() which() subset()
19. > workshop$outline[-(1:3)] 2 dialects of R: base $ [] [[]] apply() which() subset() tidyverse
20. data <- read.csv("my-data.csv")
21. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something")
22. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah")
23. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE)
24. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]
25. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]
26. data <- read.csv("my-data.csv")
27. data <- read.csv("my-data.csv") data <- data
28. data <- read.csv("my-data.csv") data <- f(data, arg1 = "something")
29. data <- read.csv("my-data.csv") data <- g( f(data, arg1 = "something"), another.thing = "blah" )
30. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )
31. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )
32. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )
33. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )
34. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )
35. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE ) data <- data[data$a.column == "cough", ]
36. %>% Solution: the pipe!
37. %>% Solution: the pipe! { } [ ] [[ ]] <- = ( ) , " " ' '
38. %>% Solution: the pipe! { } [ ] [[ ]] <- = ( ) , " " ' ' Read: “then”
39. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]
40. data
41. ↓ f() ↓ g() ↓ h() data
42. ↓ f() ↓ g() ↓ h() ↓ Some subsetting data
43. ↓ f() ↓ g() ↓ h() ↓ Some subsetting ↓ new data data
44. f(x)
45. f(x) sort(1:10)
46. f(x) sort(1:10) x %>% f()
47. f(x) sort(1:10) x %>% f() 1:10 %>% sort()
48. f(x, y) t.test(data$x, data$y)
49. f(x, y) t.test(data$x, data$y) x %>% f(y) data$x %>% t.test(data$y)
50. data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]
51. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE ) data <- data[data$a.column == "cough", ]
52. h(g(f(x)))
53. h(g(f(x))) x %>%
54. h(g(f(x))) x %>% f() %>%
55. h(g(f(x))) x %>% f() %>% g() %>%
56. h(g(f(x))) x %>% f() %>% g() %>% h() ↓ f() ↓ g() ↓ h() ↓ Some subsetting ↓ new data data
57. data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )
58. data <- read.csv("my-data.csv") data <- data %>% f(arg1 = "something") %>% g(another.thing = "blah") %>% h(a.setting = TRUE)
59. data <- read.csv("my-data.csv") data <- data %>% f(arg1 = "something") %>% g(another.thing = "blah") %>% h(a.setting = TRUE) ↓ f() ↓ g() ↓ h() ↓ Some subsetting ↓ new data data
60. data <- read.csv("my-data.csv") data <- data %>% f(arg1 = "something") %>% g(another.thing = "blah") %>% h(a.setting = TRUE) data <- data[data$a.column == "cough", ] ???????
61. > workshop$outline[1:3] DAY 1 Tidy data principles & tidyr DAY 2 Manipulating data & an intro to dplyr DAY 3 Extending your data with mutate(), summarise() & friends
62. > workshop$outline[[1]] DAY 1 Tidy data principles & tidyr
63. > workshop$outline[[1]] DAY 1 Tidy data principles & tidyr
64. A motivating example…
65. An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018
66. An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018
67. An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018
68. An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018
69. An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018
70. (A good way to collect your data!)
72. Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3
73. One way to lay out your collected data… Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3
74. Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3
75. Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 ???
76. Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 ???
77. Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 ???
78. Another way… Site 1 Site 2 Site 3 Sp
79. The “best” way. (Will make your life easiest in the long-term.) Sp Site
80. The “best” way. (Will make your life easiest in the long-term.) Sp Site TIDY DATA
81. TIDY DATA CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science
82. TIDY DATA CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science
83. TIDY DATA CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science 1. Each variable must have its own column 2. Each observation must have its own row 3. Each value, therefore, must have its own cell
84. CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science tidyr An R-package all about getting to this:
85. # Verbs to tidy your data
86. # Verbs to tidy your data # Untidy observations? gather() # if > 1 observation per row spread() # if observations live in > 1 row
87. # Verbs to tidy your data # Untidy observations? gather() # if > 1 observation per row spread() # if observations live in > 1 row # Untidy variables? separate() # if > 1 variable per column unite() # if variables live in > 1 column
88. Note the following when choosing tidyr-verbs:
89. Note the following when choosing tidyr-verbs: • Be clear on what your observations are: • Like, what unit of your study “counts” as an observation • E.g. Leaf traits: plant leaf vs plant individual • E.g. Reproductive success: egg size vs clutch size
90. Note the following when choosing tidyr-verbs: • Be clear on what your observations are: • Like, what unit of your study “counts” as an observation • E.g. Leaf traits: plant leaf vs plant individual • E.g. Reproductive success: egg size vs clutch size • This will depend on your study &/or data!
91. Note the following when choosing tidyr-verbs: • Be clear on what your observations are: • Like, what unit of your study “counts” as an observation • E.g. Leaf traits: plant leaf vs plant individual • E.g. Reproductive success: egg size vs clutch size • This will depend on your study &/or data! • Variables are discrete, separate ideas!
92. Note the following when choosing tidyr-verbs: • Be clear on what your observations are: • Like, what unit of your study “counts” as an observation • E.g. Leaf traits: plant leaf vs plant individual • E.g. Reproductive success: egg size vs clutch size • This will depend on your study &/or data! • Variables are discrete, separate ideas! • But again, this will depend on your study &/or data!
93. # Verbs to tidy your data # Untidy observations? gather() # if > 1 observation per row spread() # if observations live in > 1 row # Untidy variables? separate() # if > 1 variable per column unite() # if variables live in > 1 column
94. # Untidy observations?
95. # Untidy observations? gather() # if > 1 observation per row
96. # Untidy observations? gather() # if > 1 observation per row data %>% gather(key, value, ...)
97. # Untidy observations? gather() # if > 1 observation per row data %>% gather(key, value, ...) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
98. # Untidy observations? gather() # if > 1 observation per row data %>% gather(key, value, ...) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
99. # Untidy observations? gather() # if > 1 observation per row data %>% gather(year, cases, 1999, 2000) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
100. # Untidy observations? spread() # if observations live in > 1 row
101. # Untidy observations? spread() # if observations live in > 1 row data %>% spread(key, value)
102. # Untidy observations? spread() # if observations live in > 1 row data %>% spread(key, value)
103. # Untidy observations? spread() # if observations live in > 1 row data %>% spread(key, value) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
104. # Untidy observations? spread() # if observations live in > 1 row data %>% spread(type, count) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
105. # Untidy variables?
106. # Untidy variables? separate() # if > 1 variable per column
107. # Untidy variables? separate() # if > 1 variable per column data %>% separate(col, into, sep)
108. # Untidy variables? separate() # if > 1 variable per column data %>% separate(col, into)
109. # Untidy variables? separate() # if > 1 variable per column data %>% separate(col, into) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
110. # Untidy variables? separate() # if > 1 variable per column data %>% separate(rate, c("cases", "pop")) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
111. # Untidy variables? unite() # if variables live in > 1 column
112. # Untidy variables? unite() # if variables live in > 1 column data %>% unite(col, ..., sep)
113. # Untidy variables? unite() # if variables live in > 1 column data %>% unite(col, ...) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
114. # Untidy variables? unite() # if variables live in > 1 column data %>% unite(year, century, year) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/
115. > demo()
116. > demo() tinyurl.com/unicorns-day-1 tinyurl.com/prepost-day-1 tinyurl.com/lang-day-1 DATASETS: