Data wrangling & manipulation in R - Day 1 slides

Slide 1

Slide 1 text

data_wrangling() && ("manipulation" %in% R) postgraduate_workshop( dept = "Biological Sciences", presenter = c( "Ruan van Mazijk", "MSc candidate" ) ) %>% %>% %>% > logos() > face()

Slide 2

Slide 2 text

> introduce( )

Slide 3

Slide 3 text

> introduce( ) • BSc + Hons here at UCT

Slide 4

Slide 4 text

> introduce( ) • BSc + Hons here at UCT • Ecology & evolution • (Mostly plant) comparative biology • Biogeography

Slide 5

Slide 5 text

> introduce( ) • BSc + Hons here at UCT • Ecology & evolution • (Mostly plant) comparative biology • Biogeography • Been working with R for 4½ years • Every major project I’ve done…

Slide 6

Slide 6 text

> introduce( ) Schoenus compar Silvermine, Table Mountatin NP R. van Mazijk 2018 Tetraria ustulata Marloth NR R. van Mazijk 2018 Tetraria thermalis Silvermine, Table Mountain NP R. van Mazijk 2018

Slide 7

Slide 7 text

> workshop$goals

Slide 8

Slide 8 text

> workshop$goals • More reproducible science

Slide 9

Slide 9 text

> workshop$goals • More reproducible science • Save time by: • Automating repetitive tasks • Eliminating human error

Slide 10

Slide 10 text

> workshop$goals • More reproducible science • Save time by: • Automating repetitive tasks • Eliminating human error • Boost your skills • Think about your data programmatically

Slide 11

Slide 11 text

tinyurl.com/r-with-ruan Notes & slides will go up here: (But I encourage you to make your own notes!)

Slide 12

Slide 12 text

> workshop$outline

Slide 13

Slide 13 text

> workshop$outline[1:3]

Slide 14

Slide 14 text

> workshop$outline[1:3] DAY 1 Tidy data principles & tidyr

Slide 15

Slide 15 text

> workshop$outline[1:3] DAY 1 Tidy data principles & tidyr DAY 2 Manipulating data & an intro to dplyr DAY 3 Extending your data with mutate(), summarise() & friends

Slide 16

Slide 16 text

> workshop$outline[-(1:3)]

Slide 17

Slide 17 text

> workshop$outline[-(1:3)] 2 dialects of R:

Slide 18

Slide 18 text

> workshop$outline[-(1:3)] 2 dialects of R: base $ [] [[]] apply() which() subset()

Slide 19

Slide 19 text

> workshop$outline[-(1:3)] 2 dialects of R: base $ [] [[]] apply() which() subset() tidyverse

Slide 20

Slide 20 text

data <- read.csv("my-data.csv")

Slide 21

Slide 21 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something")

Slide 22

Slide 22 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah")

Slide 23

Slide 23 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE)

Slide 24

Slide 24 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]

Slide 25

Slide 25 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]

Slide 26

Slide 26 text

data <- read.csv("my-data.csv")

Slide 27

Slide 27 text

data <- read.csv("my-data.csv") data <- data

Slide 28

Slide 28 text

data <- read.csv("my-data.csv") data <- f(data, arg1 = "something")

Slide 29

Slide 29 text

data <- read.csv("my-data.csv") data <- g( f(data, arg1 = "something"), another.thing = "blah" )

Slide 30

Slide 30 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )

Slide 31

Slide 31 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )

Slide 32

Slide 32 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )

Slide 33

Slide 33 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )

Slide 34

Slide 34 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )

Slide 35

Slide 35 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE ) data <- data[data$a.column == "cough", ]

Slide 36

Slide 36 text

%>% Solution: the pipe!

Slide 37

Slide 37 text

%>% Solution: the pipe! { } [ ] [[ ]] <- = ( ) , " " ' '

Slide 38

Slide 38 text

%>% Solution: the pipe! { } [ ] [[ ]] <- = ( ) , " " ' ' Read: “then”

Slide 39

Slide 39 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]

Slide 40

Slide 40 text

data

Slide 41

Slide 41 text

↓ f() ↓ g() ↓ h() data

Slide 42

Slide 42 text

↓ f() ↓ g() ↓ h() ↓ Some subsetting data

Slide 43

Slide 43 text

↓ f() ↓ g() ↓ h() ↓ Some subsetting ↓ new data data

Slide 44

Slide 44 text

f(x)

Slide 45

Slide 45 text

f(x) sort(1:10)

Slide 46

Slide 46 text

f(x) sort(1:10) x %>% f()

Slide 47

Slide 47 text

f(x) sort(1:10) x %>% f() 1:10 %>% sort()

Slide 48

Slide 48 text

f(x, y) t.test(data$x, data$y)

Slide 49

Slide 49 text

f(x, y) t.test(data$x, data$y) x %>% f(y) data$x %>% t.test(data$y)

Slide 50

Slide 50 text

data <- read.csv("my-data.csv") data1 <- f(data, arg1 = "something") data2 <- g(data1, another.thing = "blah") data3 <- h(data2, a.setting = TRUE) data4 <- data3[data3$a.column == "cough", ]

Slide 51

Slide 51 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE ) data <- data[data$a.column == "cough", ]

Slide 52

Slide 52 text

h(g(f(x)))

Slide 53

Slide 53 text

h(g(f(x))) x %>%

Slide 54

Slide 54 text

h(g(f(x))) x %>% f() %>%

Slide 55

Slide 55 text

h(g(f(x))) x %>% f() %>% g() %>%

Slide 56

Slide 56 text

h(g(f(x))) x %>% f() %>% g() %>% h() ↓ f() ↓ g() ↓ h() ↓ Some subsetting ↓ new data data

Slide 57

Slide 57 text

data <- read.csv("my-data.csv") data <- h( g( f(data, arg1 = "something"), another.thing = "blah" ), a.setting = TRUE )

Slide 58

Slide 58 text

data <- read.csv("my-data.csv") data <- data %>% f(arg1 = "something") %>% g(another.thing = "blah") %>% h(a.setting = TRUE)

Slide 59

Slide 59 text

data <- read.csv("my-data.csv") data <- data %>% f(arg1 = "something") %>% g(another.thing = "blah") %>% h(a.setting = TRUE) ↓ f() ↓ g() ↓ h() ↓ Some subsetting ↓ new data data

Slide 60

Slide 60 text

data <- read.csv("my-data.csv") data <- data %>% f(arg1 = "something") %>% g(another.thing = "blah") %>% h(a.setting = TRUE) data <- data[data$a.column == "cough", ] ???????

Slide 61

Slide 61 text

> workshop$outline[1:3] DAY 1 Tidy data principles & tidyr DAY 2 Manipulating data & an intro to dplyr DAY 3 Extending your data with mutate(), summarise() & friends

Slide 62

Slide 62 text

> workshop$outline[[1]] DAY 1 Tidy data principles & tidyr

Slide 63

Slide 63 text

> workshop$outline[[1]] DAY 1 Tidy data principles & tidyr

Slide 64

Slide 64 text

A motivating example…

Slide 65

Slide 65 text

An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018

Slide 66

Slide 66 text

An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018

Slide 67

Slide 67 text

An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018

Slide 68

Slide 68 text

An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018

Slide 69

Slide 69 text

An example data-collection scenario in biology Kogelberg NR, R. van Mazijk 2019 Observation Pk, R. van Mazijk 2018 Near Pearly Beach, Agulhas Plains, R. van Mazijk 2018

Slide 70

Slide 70 text

(A good way to collect your data!)

Slide 71

Slide 71 text

No content

Slide 72

Slide 72 text

Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3

Slide 73

Slide 73 text

One way to lay out your collected data… Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3

Slide 74

Slide 74 text

Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3

Slide 75

Slide 75 text

Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 ???

Slide 76

Slide 76 text

Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 ???

Slide 77

Slide 77 text

Site 1 Site 2 Site 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 Sp 1 Sp 2 Sp 3 ???

Slide 78

Slide 78 text

Another way… Site 1 Site 2 Site 3 Sp

Slide 79

Slide 79 text

The “best” way. (Will make your life easiest in the long-term.) Sp Site

Slide 80

Slide 80 text

The “best” way. (Will make your life easiest in the long-term.) Sp Site TIDY DATA

Slide 81

Slide 81 text

TIDY DATA CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science

Slide 82

Slide 82 text

TIDY DATA CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science

Slide 83

Slide 83 text

TIDY DATA CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science 1. Each variable must have its own column 2. Each observation must have its own row 3. Each value, therefore, must have its own cell

Slide 84

Slide 84 text

CC BY-NC-ND 3.0 Grolemund & Wickham 2017. R for Data Science tidyr An R-package all about getting to this:

Slide 85

Slide 85 text

# Verbs to tidy your data

Slide 86

Slide 86 text

# Verbs to tidy your data # Untidy observations? gather() # if > 1 observation per row spread() # if observations live in > 1 row

Slide 87

Slide 87 text

# Verbs to tidy your data # Untidy observations? gather() # if > 1 observation per row spread() # if observations live in > 1 row # Untidy variables? separate() # if > 1 variable per column unite() # if variables live in > 1 column

Slide 88

Slide 88 text

Note the following when choosing tidyr-verbs:

Slide 89

Slide 89 text

Slide 90

Slide 90 text

Note the following when choosing tidyr-verbs: • Be clear on what your observations are: • Like, what unit of your study “counts” as an observation • E.g. Leaf traits: plant leaf vs plant individual • E.g. Reproductive success: egg size vs clutch size • This will depend on your study &/or data!

Slide 91

Slide 91 text

Slide 92

Slide 92 text

Slide 93

Slide 93 text

Slide 94

Slide 94 text

# Untidy observations?

Slide 95

Slide 95 text

# Untidy observations? gather() # if > 1 observation per row

Slide 96

Slide 96 text

# Untidy observations? gather() # if > 1 observation per row data %>% gather(key, value, ...)

Slide 97

Slide 97 text

# Untidy observations? gather() # if > 1 observation per row data %>% gather(key, value, ...) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 98

Slide 98 text

# Untidy observations? gather() # if > 1 observation per row data %>% gather(key, value, ...) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 99

Slide 99 text

# Untidy observations? gather() # if > 1 observation per row data %>% gather(year, cases, 1999, 2000) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 100

Slide 100 text

# Untidy observations? spread() # if observations live in > 1 row

Slide 101

Slide 101 text

# Untidy observations? spread() # if observations live in > 1 row data %>% spread(key, value)

Slide 102

Slide 102 text

# Untidy observations? spread() # if observations live in > 1 row data %>% spread(key, value)

Slide 103

Slide 103 text

# Untidy observations? spread() # if observations live in > 1 row data %>% spread(key, value) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 104

Slide 104 text

# Untidy observations? spread() # if observations live in > 1 row data %>% spread(type, count) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 105

Slide 105 text

# Untidy variables?

Slide 106

Slide 106 text

# Untidy variables? separate() # if > 1 variable per column

Slide 107

Slide 107 text

# Untidy variables? separate() # if > 1 variable per column data %>% separate(col, into, sep)

Slide 108

Slide 108 text

# Untidy variables? separate() # if > 1 variable per column data %>% separate(col, into)

Slide 109

Slide 109 text

# Untidy variables? separate() # if > 1 variable per column data %>% separate(col, into) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 110

Slide 110 text

# Untidy variables? separate() # if > 1 variable per column data %>% separate(rate, c("cases", "pop")) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 111

Slide 111 text

# Untidy variables? unite() # if variables live in > 1 column

Slide 112

Slide 112 text

# Untidy variables? unite() # if variables live in > 1 column data %>% unite(col, ..., sep)

Slide 113

Slide 113 text

# Untidy variables? unite() # if variables live in > 1 column data %>% unite(col, ...) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 114

Slide 114 text

# Untidy variables? unite() # if variables live in > 1 column data %>% unite(year, century, year) CC BY SA RStudio https://www.rstudio.com/resources/cheatsheets/

Slide 115

Slide 115 text

> demo()

Slide 116

Slide 116 text

> demo() tinyurl.com/unicorns-day-1 tinyurl.com/prepost-day-1 tinyurl.com/lang-day-1 DATASETS: