Hacking Drone with Golang

Talk about OpenCV (GoCV) and Gobot framework.

Transcript

1. Hacking Drone with Golang jOpenSpace 2018
2. Who am I? 4 Ladislav Prskavec (@abtris) 4 Developer & SRE in Oracle Apiary 4 Ruby & Node.js & Go 4 Follow Golang meetup on twitter @GoMeetupPrague
3. Drone 4 Ryze Tello 4 Intel Myriad VPU 4 5MPx camera (720p30, 2592x1936), 80g 4 Electronic stabilisation 4 no GPS
4. OpenCV 4 Open Source Computer Vision Library 4 OpenCV was built to provide a common infrastructure for computer vision applications and to accelerate the use of machine perception in the commercial products. 4 GoCV
5. Why you should use Go for computer vision 4 Concurrency 4 Portability 4 Performance
6. Gobot 4 framework for robots, drones, and the Internet of Things (IoT)
7. Hello World with Gobot and Drone Tello
8. package main import ( "fmt" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" ) func main() { drone := tello.NewDriver("8888") var flightData *tello.FlightData var battery int8 work := func() { drone.TakeOff() drone.On(tello.FlightDataEvent, func(data interface{}) { flightData = data.(*tello.FlightData) battery = flightData.BatteryPercentage fmt.Println("Height:", flightData.Height) }) gobot.After(5*time.Second, func() { fmt.Println("Battery:", battery) drone.Land() }) } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) robot.Start() }
9. import ( "fmt" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" )
10. func main() { drone := tello.NewDriver("8888") work := func() { ... } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) robot.Start() }
11. work := func() { drone.TakeOff() drone.On(tello.FlightDataEvent, func(data interface{}) { flightData = data.(*tello.FlightData) battery = flightData.BatteryPercentage fmt.Println("Height:", flightData.Height) }) gobot.After(5*time.Second, func() { fmt.Println("Battery:", battery) drone.Land() }) }
12. DEMO $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello
13. Face detect
14. 4 CascadeClassifier 4 external data file containing the classifier data (haarcascade_frontalface_def ault.xml)
15. Others GoCV examples 4 Caffe Classifier 4 Motion-detect 4 Pose 4 SSD Facedetect (Advanced Deep Neural Network example that uses SSD classifier) 4 TF Classifier (Tensorflow)
16. OpenCV with DJI Tello
17. package main import ( "fmt" "image" "image/color" "io" "os/exec" "strconv" "time" "gobot.io/x/gobot" "gobot.io/x/gobot/platforms/dji/tello" "gocv.io/x/gocv" )
18. const ( frameX = 400 frameY = 300 frameSize = frameX * frameY * 3 ) func main() { drone := tello.NewDriver("8890") window := gocv.NewWindow("Tello") xmlFile := "haarcascade_frontalface_default.xml" ffmpeg := exec.Command("ffmpeg", "-hwaccel", "auto", "-hwaccel_device", "opencl", "-i", "pipe:0", "-pix_fmt", "bgr24", "-s", strconv.Itoa(frameX)+"x"+strconv.Itoa(frameY), "-f", "rawvideo", "pipe:1") ffmpegIn, _ := ffmpeg.StdinPipe() ffmpegOut, _ := ffmpeg.StdoutPipe()
19. work := func() { if err := ffmpeg.Start(); err != nil { fmt.Println(err) return } drone.On(tello.ConnectedEvent, func(data interface{}) { fmt.Println("Connected") drone.StartVideo() drone.SetVideoEncoderRate(tello.VideoBitRateAuto) drone.SetExposure(0) gobot.Every(100*time.Millisecond, func() { drone.StartVideo() }) }) drone.On(tello.VideoFrameEvent, func(data interface{}) { pkt := data.([]byte) if _, err := ffmpegIn.Write(pkt); err != nil { fmt.Println(err) } }) } robot := gobot.NewRobot("tello", []gobot.Connection{}, []gobot.Device{drone}, work, ) // calling Start(false) lets the Start routine return immediately without an additional blocking goroutine robot.Start(false)
20. // now handle video frames from ffmpeg stream in main thread, to be macOS/Windows friendly for { buf := make([]byte, frameSize) if _, err := io.ReadFull(ffmpegOut, buf); err != nil { fmt.Println(err) continue } img, _ := gocv.NewMatFromBytes(frameY, frameX, gocv.MatTypeCV8UC3, buf) if img.Empty() { continue } // detect faces // color for the rect when faces detected blue := color.RGBA{0, 0, 255, 0} // load classifier to recognize faces classifier := gocv.NewCascadeClassifier() defer classifier.Close() if !classifier.Load(xmlFile) { fmt.Printf("Error reading cascade file: %v\n", xmlFile) return } rects := classifier.DetectMultiScale(img) fmt.Printf("found %d faces\n", len(rects)) // draw a rectangle around each face on the original image, // along with text identifying as "Human" for _, r := range rects { gocv.Rectangle(&img, r, blue, 3) size := gocv.GetTextSize("Human", gocv.FontHersheyPlain, 1.2, 2) pt := image.Pt(r.Min.X+(r.Min.X/2)-(size.X/2), r.Min.Y-2) gocv.PutText(&img, "Human", pt, gocv.FontHersheyPlain, 1.2, blue, 2) } window.IMShow(img) if window.WaitKey(1) >= 0 { break } } }
21. DEMO $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello- opencv-keyboard