Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Hacking Drone with Golang

Hacking Drone with Golang

Talk about OpenCV (GoCV) and Gobot framework.

Ladislav Prskavec

October 06, 2018
Tweet

More Decks by Ladislav Prskavec

Other Decks in Technology

Transcript

  1. Hacking Drone with Golang
    jOpenSpace 2018

    View Slide

  2. Who am I?
    4 Ladislav Prskavec (@abtris)
    4 Developer & SRE in Oracle Apiary
    4 Ruby & Node.js & Go
    4 Follow Golang meetup on twitter
    @GoMeetupPrague

    View Slide

  3. Drone
    4 Ryze Tello
    4 Intel Myriad VPU
    4 5MPx camera (720p30, 2592x1936), 80g
    4 Electronic stabilisation
    4 no GPS

    View Slide

  4. OpenCV
    4 Open Source Computer Vision Library
    4 OpenCV was built to provide a common
    infrastructure for computer vision applications and
    to accelerate the use of machine perception in the
    commercial products.
    4 GoCV

    View Slide

  5. Why you should use Go
    for computer vision
    4 Concurrency
    4 Portability
    4 Performance

    View Slide

  6. Gobot
    4 framework for robots, drones, and the Internet of
    Things (IoT)

    View Slide

  7. Hello World with Gobot
    and Drone Tello

    View Slide

  8. package main
    import (
    "fmt"
    "time"
    "gobot.io/x/gobot"
    "gobot.io/x/gobot/platforms/dji/tello"
    )
    func main() {
    drone := tello.NewDriver("8888")
    var flightData *tello.FlightData
    var battery int8
    work := func() {
    drone.TakeOff()
    drone.On(tello.FlightDataEvent, func(data interface{}) {
    flightData = data.(*tello.FlightData)
    battery = flightData.BatteryPercentage
    fmt.Println("Height:", flightData.Height)
    })
    gobot.After(5*time.Second, func() {
    fmt.Println("Battery:", battery)
    drone.Land()
    })
    }
    robot := gobot.NewRobot("tello",
    []gobot.Connection{},
    []gobot.Device{drone},
    work,
    )
    robot.Start()
    }

    View Slide

  9. import (
    "fmt"
    "time"
    "gobot.io/x/gobot"
    "gobot.io/x/gobot/platforms/dji/tello"
    )

    View Slide

  10. func main() {
    drone := tello.NewDriver("8888")
    work := func() {
    ...
    }
    robot := gobot.NewRobot("tello",
    []gobot.Connection{},
    []gobot.Device{drone},
    work,
    )
    robot.Start()
    }

    View Slide

  11. work := func() {
    drone.TakeOff()
    drone.On(tello.FlightDataEvent, func(data interface{}) {
    flightData = data.(*tello.FlightData)
    battery = flightData.BatteryPercentage
    fmt.Println("Height:", flightData.Height)
    })
    gobot.After(5*time.Second, func() {
    fmt.Println("Battery:", battery)
    drone.Land()
    })
    }

    View Slide

  12. DEMO
    $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello

    View Slide

  13. Face detect

    View Slide

  14. 4 CascadeClassifier
    4 external data file containing
    the classifier data
    (haarcascade_frontalface_def
    ault.xml)

    View Slide

  15. Others GoCV examples
    4 Caffe Classifier
    4 Motion-detect
    4 Pose
    4 SSD Facedetect (Advanced Deep Neural Network
    example that uses SSD classifier)
    4 TF Classifier (Tensorflow)

    View Slide

  16. OpenCV with DJI Tello

    View Slide

  17. package main
    import (
    "fmt"
    "image"
    "image/color"
    "io"
    "os/exec"
    "strconv"
    "time"
    "gobot.io/x/gobot"
    "gobot.io/x/gobot/platforms/dji/tello"
    "gocv.io/x/gocv"
    )

    View Slide

  18. const (
    frameX = 400
    frameY = 300
    frameSize = frameX * frameY * 3
    )
    func main() {
    drone := tello.NewDriver("8890")
    window := gocv.NewWindow("Tello")
    xmlFile := "haarcascade_frontalface_default.xml"
    ffmpeg := exec.Command("ffmpeg", "-hwaccel", "auto", "-hwaccel_device", "opencl", "-i", "pipe:0",
    "-pix_fmt", "bgr24", "-s", strconv.Itoa(frameX)+"x"+strconv.Itoa(frameY), "-f", "rawvideo", "pipe:1")
    ffmpegIn, _ := ffmpeg.StdinPipe()
    ffmpegOut, _ := ffmpeg.StdoutPipe()

    View Slide

  19. work := func() {
    if err := ffmpeg.Start(); err != nil {
    fmt.Println(err)
    return
    }
    drone.On(tello.ConnectedEvent, func(data interface{}) {
    fmt.Println("Connected")
    drone.StartVideo()
    drone.SetVideoEncoderRate(tello.VideoBitRateAuto)
    drone.SetExposure(0)
    gobot.Every(100*time.Millisecond, func() {
    drone.StartVideo()
    })
    })
    drone.On(tello.VideoFrameEvent, func(data interface{}) {
    pkt := data.([]byte)
    if _, err := ffmpegIn.Write(pkt); err != nil {
    fmt.Println(err)
    }
    })
    }
    robot := gobot.NewRobot("tello",
    []gobot.Connection{},
    []gobot.Device{drone},
    work,
    )
    // calling Start(false) lets the Start routine return immediately without an additional blocking goroutine
    robot.Start(false)

    View Slide

  20. // now handle video frames from ffmpeg stream in main thread, to be macOS/Windows friendly
    for {
    buf := make([]byte, frameSize)
    if _, err := io.ReadFull(ffmpegOut, buf); err != nil {
    fmt.Println(err)
    continue
    }
    img, _ := gocv.NewMatFromBytes(frameY, frameX, gocv.MatTypeCV8UC3, buf)
    if img.Empty() {
    continue
    }
    // detect faces
    // color for the rect when faces detected
    blue := color.RGBA{0, 0, 255, 0}
    // load classifier to recognize faces
    classifier := gocv.NewCascadeClassifier()
    defer classifier.Close()
    if !classifier.Load(xmlFile) {
    fmt.Printf("Error reading cascade file: %v\n", xmlFile)
    return
    }
    rects := classifier.DetectMultiScale(img)
    fmt.Printf("found %d faces\n", len(rects))
    // draw a rectangle around each face on the original image,
    // along with text identifying as "Human"
    for _, r := range rects {
    gocv.Rectangle(&img, r, blue, 3)
    size := gocv.GetTextSize("Human", gocv.FontHersheyPlain, 1.2, 2)
    pt := image.Pt(r.Min.X+(r.Min.X/2)-(size.X/2), r.Min.Y-2)
    gocv.PutText(&img, "Human", pt, gocv.FontHersheyPlain, 1.2, blue, 2)
    }
    window.IMShow(img)
    if window.WaitKey(1) >= 0 {
    break
    }
    }
    }

    View Slide

  21. DEMO
    $GOPATH/go/src/github.com/abtris/dji-tello/workshop/hello-tello-
    opencv-keyboard

    View Slide