Hacking Drone with Golang

Talk about OpenCV (GoCV) and Gobot framework.

Transcript

1. Hacking Drone with Golang
   jOpenSpace 2018
   

   View Slide

2. Who am I?
   4 Ladislav Prskavec (@abtris)
   4 Developer & SRE in Oracle Apiary
   4 Ruby & Node.js & Go
   4 Follow Golang meetup on twitter
   @GoMeetupPrague
   

   View Slide

3. Drone
   4 Ryze Tello
   4 Intel Myriad VPU
   4 5MPx camera (720p30, 2592x1936), 80g
   4 Electronic stabilisation
   4 no GPS
   

   View Slide

4. OpenCV
   4 Open Source Computer Vision Library
   4 OpenCV was built to provide a common
   infrastructure for computer vision applications and
   to accelerate the use of machine perception in the
   commercial products.
   4 GoCV
   

   View Slide

5. Why you should use Go
   for computer vision
   4 Concurrency
   4 Portability
   4 Performance
   

   View Slide

6. Gobot
   4 framework for robots, drones, and the Internet of
   Things (IoT)
   

   View Slide

7. Hello World with Gobot
   and Drone Tello
   

   View Slide

8. package main
   import (
   "fmt"
   "time"
   "gobot.io/x/gobot"
   "gobot.io/x/gobot/platforms/dji/tello"
   )
   func main() {
   drone := tello.NewDriver("8888")
   var flightData *tello.FlightData
   var battery int8
   work := func() {
   drone.TakeOff()
   drone.On(tello.FlightDataEvent, func(data interface{}) {
   flightData = data.(*tello.FlightData)
   battery = flightData.BatteryPercentage
   fmt.Println("Height:", flightData.Height)
   })
   gobot.After(5*time.Second, func() {
   fmt.Println("Battery:", battery)
   drone.Land()
   })
   }
   robot := gobot.NewRobot("tello",
   []gobot.Connection{},
   []gobot.Device{drone},
   work,
   )
   robot.Start()
   }
   

   View Slide

9. import (
   "fmt"
   "time"
   "gobot.io/x/gobot"
   "gobot.io/x/gobot/platforms/dji/tello"
   )
   

   View Slide

10. func main() {
    drone := tello.NewDriver("8888")
    work := func() {
    ...
    }
    robot := gobot.NewRobot("tello",
    []gobot.Connection{},
    []gobot.Device{drone},
    work,
    )
    robot.Start()
    }
    

    View Slide

11. work := func() {
    drone.TakeOff()
    drone.On(tello.FlightDataEvent, func(data interface{}) {
    flightData = data.(*tello.FlightData)
    battery = flightData.BatteryPercentage
    fmt.Println("Height:", flightData.Height)
    })
    gobot.After(5*time.Second, func() {
    fmt.Println("Battery:", battery)
    drone.Land()
    })
    }
    

    View Slide

12. DEMO
    $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello
    

    View Slide

13. Face detect
    

    View Slide

14. 4 CascadeClassifier
    4 external data file containing
    the classifier data
    (haarcascade_frontalface_def
    ault.xml)
    

    View Slide

15. Others GoCV examples
    4 Caffe Classifier
    4 Motion-detect
    4 Pose
    4 SSD Facedetect (Advanced Deep Neural Network
    example that uses SSD classifier)
    4 TF Classifier (Tensorflow)
    

    View Slide

16. OpenCV with DJI Tello
    

    View Slide

17. package main
    import (
    "fmt"
    "image"
    "image/color"
    "io"
    "os/exec"
    "strconv"
    "time"
    "gobot.io/x/gobot"
    "gobot.io/x/gobot/platforms/dji/tello"
    "gocv.io/x/gocv"
    )
    

    View Slide

18. const (
    frameX = 400
    frameY = 300
    frameSize = frameX * frameY * 3
    )
    func main() {
    drone := tello.NewDriver("8890")
    window := gocv.NewWindow("Tello")
    xmlFile := "haarcascade_frontalface_default.xml"
    ffmpeg := exec.Command("ffmpeg", "-hwaccel", "auto", "-hwaccel_device", "opencl", "-i", "pipe:0",
    "-pix_fmt", "bgr24", "-s", strconv.Itoa(frameX)+"x"+strconv.Itoa(frameY), "-f", "rawvideo", "pipe:1")
    ffmpegIn, _ := ffmpeg.StdinPipe()
    ffmpegOut, _ := ffmpeg.StdoutPipe()
    

    View Slide

19. work := func() {
    if err := ffmpeg.Start(); err != nil {
    fmt.Println(err)
    return
    }
    drone.On(tello.ConnectedEvent, func(data interface{}) {
    fmt.Println("Connected")
    drone.StartVideo()
    drone.SetVideoEncoderRate(tello.VideoBitRateAuto)
    drone.SetExposure(0)
    gobot.Every(100*time.Millisecond, func() {
    drone.StartVideo()
    })
    })
    drone.On(tello.VideoFrameEvent, func(data interface{}) {
    pkt := data.([]byte)
    if _, err := ffmpegIn.Write(pkt); err != nil {
    fmt.Println(err)
    }
    })
    }
    robot := gobot.NewRobot("tello",
    []gobot.Connection{},
    []gobot.Device{drone},
    work,
    )
    // calling Start(false) lets the Start routine return immediately without an additional blocking goroutine
    robot.Start(false)
    

    View Slide

20. // now handle video frames from ffmpeg stream in main thread, to be macOS/Windows friendly
    for {
    buf := make([]byte, frameSize)
    if _, err := io.ReadFull(ffmpegOut, buf); err != nil {
    fmt.Println(err)
    continue
    }
    img, _ := gocv.NewMatFromBytes(frameY, frameX, gocv.MatTypeCV8UC3, buf)
    if img.Empty() {
    continue
    }
    // detect faces
    // color for the rect when faces detected
    blue := color.RGBA{0, 0, 255, 0}
    // load classifier to recognize faces
    classifier := gocv.NewCascadeClassifier()
    defer classifier.Close()
    if !classifier.Load(xmlFile) {
    fmt.Printf("Error reading cascade file: %v\n", xmlFile)
    return
    }
    rects := classifier.DetectMultiScale(img)
    fmt.Printf("found %d faces\n", len(rects))
    // draw a rectangle around each face on the original image,
    // along with text identifying as "Human"
    for _, r := range rects {
    gocv.Rectangle(&img, r, blue, 3)
    size := gocv.GetTextSize("Human", gocv.FontHersheyPlain, 1.2, 2)
    pt := image.Pt(r.Min.X+(r.Min.X/2)-(size.X/2), r.Min.Y-2)
    gocv.PutText(&img, "Human", pt, gocv.FontHersheyPlain, 1.2, blue, 2)
    }
    window.IMShow(img)
    if window.WaitKey(1) >= 0 {
    break
    }
    }
    }
    

    View Slide

21. DEMO
    $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello-
    opencv-keyboard
    

    View Slide