Slide 1

Slide 1 text

Hacking Drone with Golang jOpenSpace 2018

Slide 2

Slide 2 text

Who am I? 4 Ladislav Prskavec (@abtris) 4 Developer & SRE in Oracle Apiary 4 Ruby & Node.js & Go 4 Follow Golang meetup on twitter @GoMeetupPrague

Slide 3

Slide 3 text

Drone 4 Ryze Tello 4 Intel Myriad VPU 4 5MPx camera (720p30, 2592x1936), 80g 4 Electronic stabilisation 4 no GPS

Slide 4

Slide 4 text

OpenCV 4 Open Source Computer Vision Library 4 OpenCV was built to provide a common infrastructure for computer vision applications and to accelerate the use of machine perception in the commercial products. 4 GoCV

Slide 5

Slide 5 text

Why you should use Go for computer vision 4 Concurrency 4 Portability 4 Performance

Slide 6

Slide 6 text

Gobot 4 framework for robots, drones, and the Internet of Things (IoT)

Slide 7

Slide 7 text

Hello World with Gobot and Drone Tello

Slide 8

Slide 8 text

package main import ( "fmt" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" ) func main() { drone := tello.NewDriver("8888") var flightData *tello.FlightData var battery int8 work := func() { drone.TakeOff() drone.On(tello.FlightDataEvent, func(data interface{}) { flightData = data.(*tello.FlightData) battery = flightData.BatteryPercentage fmt.Println("Height:", flightData.Height) }) gobot.After(5*time.Second, func() { fmt.Println("Battery:", battery) drone.Land() }) } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) robot.Start() }

Slide 9

Slide 9 text

import ( "fmt" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" )

Slide 10

Slide 10 text

func main() { drone := tello.NewDriver("8888") work := func() { ... } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) robot.Start() }

Slide 11

Slide 11 text

work := func() { drone.TakeOff() drone.On(tello.FlightDataEvent, func(data interface{}) { flightData = data.(*tello.FlightData) battery = flightData.BatteryPercentage fmt.Println("Height:", flightData.Height) }) gobot.After(5*time.Second, func() { fmt.Println("Battery:", battery) drone.Land() }) }

Slide 12

Slide 12 text

DEMO $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello

Slide 13

Slide 13 text

Face detect

Slide 14

Slide 14 text

4 CascadeClassifier 4 external data file containing the classifier data (haarcascade_frontalface_def ault.xml)

Slide 15

Slide 15 text

Others GoCV examples 4 Caffe Classifier 4 Motion-detect 4 Pose 4 SSD Facedetect (Advanced Deep Neural Network example that uses SSD classifier) 4 TF Classifier (Tensorflow)

Slide 16

Slide 16 text

OpenCV with DJI Tello

Slide 17

Slide 17 text

package main import ( "fmt" "image" "image/color" "io" "os/exec" "strconv" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" "gocv.io/x/gocv" )

Slide 18

Slide 18 text

const ( frameX = 400 frameY = 300 frameSize = frameX * frameY * 3 ) func main() { drone := tello.NewDriver("8890") window := gocv.NewWindow("Tello") xmlFile := "haarcascade_frontalface_default.xml" ffmpeg := exec.Command("ffmpeg", "-hwaccel", "auto", "-hwaccel_device", "opencl", "-i", "pipe:0", "-pix_fmt", "bgr24", "-s", strconv.Itoa(frameX)+"x"+strconv.Itoa(frameY), "-f", "rawvideo", "pipe:1") ffmpegIn, _ := ffmpeg.StdinPipe() ffmpegOut, _ := ffmpeg.StdoutPipe()

Slide 19

Slide 19 text

work := func() { if err := ffmpeg.Start(); err != nil { fmt.Println(err) return } drone.On(tello.ConnectedEvent, func(data interface{}) { fmt.Println("Connected") drone.StartVideo() drone.SetVideoEncoderRate(tello.VideoBitRateAuto) drone.SetExposure(0) gobot.Every(100*time.Millisecond, func() { drone.StartVideo() }) }) drone.On(tello.VideoFrameEvent, func(data interface{}) { pkt := data.([]byte) if _, err := ffmpegIn.Write(pkt); err != nil { fmt.Println(err) } }) } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) // calling Start(false) lets the Start routine return immediately without an additional blocking goroutine robot.Start(false)

Slide 20

Slide 20 text

// now handle video frames from ffmpeg stream in main thread, to be macOS/Windows friendly for { buf := make([]byte, frameSize) if _, err := io.ReadFull(ffmpegOut, buf); err != nil { fmt.Println(err) continue } img, _ := gocv.NewMatFromBytes(frameY, frameX, gocv.MatTypeCV8UC3, buf) if img.Empty() { continue } // detect faces // color for the rect when faces detected blue := color.RGBA{0, 0, 255, 0} // load classifier to recognize faces classifier := gocv.NewCascadeClassifier() defer classifier.Close() if !classifier.Load(xmlFile) { fmt.Printf("Error reading cascade file: %v\n", xmlFile) return } rects := classifier.DetectMultiScale(img) fmt.Printf("found %d faces\n", len(rects)) // draw a rectangle around each face on the original image, // along with text identifying as "Human" for _, r := range rects { gocv.Rectangle(&img, r, blue, 3) size := gocv.GetTextSize("Human", gocv.FontHersheyPlain, 1.2, 2) pt := image.Pt(r.Min.X+(r.Min.X/2)-(size.X/2), r.Min.Y-2) gocv.PutText(&img, "Human", pt, gocv.FontHersheyPlain, 1.2, blue, 2) } window.IMShow(img) if window.WaitKey(1) >= 0 { break } } }

Slide 21

Slide 21 text

DEMO $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello- opencv-keyboard