Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Getting Started with Azure Kinect DK

Getting Started with Azure Kinect DK

Slice deck of tutorial of contents creation with Azure Kinect and Unity.

TakashiYoshinaga

September 08, 2019
Tweet

More Decks by TakashiYoshinaga

Other Decks in Technology

Transcript

  1. Getting Started with
    Azure Kinect

    View full-size slide

  2. Download Pre-Build SDK and Asset
    http://arfukuoka.lolipop.jp/Azure
    Kinect/Sample.zip

    View full-size slide

  3. Goal of the Tutorial
    Visualization of Colored Point Cloud on AR marker

    View full-size slide

  4. Creating Project
    New

    View full-size slide

  5. Creating Project
    Create Project
    Project Name
    Directory

    View full-size slide

  6. Switching .NET Environment
    ①File
    ②Build Setting

    View full-size slide

  7. Switching .NET Environment
    Player Settings

    View full-size slide

  8. Switching .NET Environment
    Other Settings

    View full-size slide

  9. Switching .NET Environment
    Scripting Runtime Versionを
    .NET 4.X Equivalent
    Api Compatibility Levelを
    .NET 4.X

    View full-size slide

  10. Installation of Azure Kinect SDK
    Right ClickAssets

    View full-size slide

  11. Installation of Azure Kinect SDK
    ①Create
    ②Folder

    View full-size slide

  12. Installation of Azure Kinect SDK
    Rename it to Plugins

    View full-size slide

  13. Installation of Azure Kinect SDK
    Click Plugins

    View full-size slide

  14. Installation of Azure Kinect SDK
    ①Open following directory by explorer
    Sample¥Plugins
    ②Drag & Drop all Files
    into Plugins of this project

    View full-size slide

  15. Installation of Azure Kinect SDK
    ①Open the directory of which Azure Kinect SDK
    is installed
    C:¥Program Files¥Azure Kinect SDK
    v1.2.0¥sdk¥windows-desktop¥amd64¥release¥bin
    ②Drag and drop depthengine_2_0.dll
    into Plugins of your project

    View full-size slide

  16. Adding Object to Visualize Point Cloud
    Right Click

    View full-size slide

  17. Adding Object to Visualize Point Cloud
    Create Empty

    View full-size slide

  18. Adding Object to Visualize Point Cloud
    ①GameObject
    ②Rename into PointCloud

    View full-size slide

  19. Adding Object to Visualize Point Cloud
    Add Component

    View full-size slide

  20. Adding Object to Visualize Point Cloud
    ①seach ”mesh”
    ②Mesh Filter

    View full-size slide

  21. Adding Object to Visualize Point Cloud
    Add Component

    View full-size slide

  22. Adding Object to Visualize Point Cloud
    Mesh Renderer

    View full-size slide

  23. Adding Object to Visualize Point Cloud
    Open Materials

    View full-size slide

  24. Adding Object to Visualize Point Cloud
    Next step, material for drawing
    point cloud will be set to Element0

    View full-size slide

  25. Adding a Material to Draw Point Cloud
    ①Sample Folder
    ②Double Click PointCloud
    ③Import

    View full-size slide

  26. Adding a Material to Draw Point Cloud
    ①Assets
    ②Check ColoredVertex & PointCloud are added

    View full-size slide

  27. Adding a Material to Draw Point Cloud
    ①PointCloud
    ②Drag & Drop PointCloud
    into Element0

    View full-size slide

  28. Adding a Material to Draw Point Cloud
    Custom/ColoredVertex

    View full-size slide

  29. Adding a Script to Draw Point Cloud
    ①Point Cloud
    ②Add Component

    View full-size slide

  30. Adding a Script to Draw Point Cloud
    ①Remove search word
    ②New Script
    ③KinectScript
    ④Create and Add

    View full-size slide

  31. Adding a Script to Draw Point Cloud
    Check Script is added

    View full-size slide

  32. Adding a Script to Draw Point Cloud
    Double Click KinectScript
    to open this script

    View full-size slide

  33. Adding a Script to Draw Point Cloud

    View full-size slide

  34. Start and Stop Azure Kinect
    using UnityEngine;
    using Microsoft.Azure.Kinect.Sensor;
    public class KinectScript : MonoBehaviour
    {
    Device device; //Variable to access to Kinect
    void Start()
    {
    InitKinect(); //Initialization of Kinect
    }
    void InitKinect()
    {
    //See Next Page.
    }
    /*Omitted */
    }

    View full-size slide

  35. Adding a Script to Draw Point Cloud
    void InitKinect()
    {
    device = Device.Open(0); //OpenKinect
    device.StartCameras(new DeviceConfiguration
    {
    ColorFormat = ImageFormat.ColorBGRA32,
    ColorResolution = ColorResolution.R720p,
    DepthMode = DepthMode.NFOV_Unbinned,
    SynchronizedImagesOnly = true,
    CameraFPS = FPS.FPS30,
    });
    }
    void OnDestroy() //Stop Kinect
    {
    device.StopCameras();
    }

    View full-size slide

  36. Run
    LED is turned on

    View full-size slide

  37. Run
    Stop Play again

    View full-size slide

  38. Preparation of Drawing Point Cloud
    Device device;
    int width, height, num; //width, height and total Pixels
    void Start(){ /*Omitted*/ }
    void InitKinect()
    {
    device = Device.Open(0);
    device.StartCameras ( /*Omitted*/ );
    //Calculation of total number of pixel.
    width = device.GetCalibration().
    depth_camera_calibration.resolution_width;
    height = device.GetCalibration().
    depth_camera_calibration.resolution_height;
    num = height * width;
    }

    View full-size slide

  39. Preparation of Drawing Point Cloud
    int width, height, num;
    Mesh mesh; //Used for Drawing Shape
    Vector3[] vertices; //Position of each point.
    Color32[] colors; //Color of each point.
    int[] indices; //Index list of drawing point
    void Start()
    {
    InitKinect();
    InitMesh(); //Initialization of mesh
    }
    void InitMesh()
    {
    //See the next page.
    }

    View full-size slide

  40. Preparation of Drawing Point Cloud
    void InitMesh()
    {
    mesh = new Mesh();
    //Enable drawing more than 65535 points
    mesh.indexFormat =
    UnityEngine.Rendering.IndexFormat.UInt32;
    //allocation of memory for treating information to
    //draw point cloud.
    vertices = new Vector3[num];
    colors = new Color32[num];
    indices = new int[num];
    //See next page
    }

    View full-size slide

  41. Next Step
    Draw points as many number as pixel of depth image.

    View full-size slide

  42. Drawing Points
    /*Continued from previous step (in InitMesh function)*/
    int index = 0; //index of vertex array.
    for (int y = 0; y < height; y++)
    {
    for (int x = 0; x < width; x++)
    {
    vertices[index].x = x ;
    vertices[index].y = y ;
    vertices[index].z = 2;
    colors[index].r = 0;
    colors [index].g = 0;
    colors [index].b = 255;
    colors [index].a = 255;
    indices[index] = index;
    index++;
    }
    }
    Temporally setting the
    coordinates of each vertex
    Temporally setting the
    color of each vertex
    Setting drawing list.

    View full-size slide

  43. Drawing Points
    /*Continued from the previous page.*/
    int index = 0;
    for (int y = 0; y < height; y++)
    {
    for (int x = 0; x < width; x++)
    {
    /*Omitted*/
    }
    }
    //Assignment vertices and colors to mesh.
    mesh.vertices = vertices;
    mesh.colors32 = colors;
    //Setting drawing list and mesh topology.
    mesh.SetIndices(indices, MeshTopology.Points, 0);
    //Assingnment of mesh to MeshFilter of the gameObject.
    gameObject.GetComponent().mesh = mesh;

    View full-size slide

  44. Run
    Camera is too close to points

    View full-size slide

  45. Run
    ①Scene Tab
    ②Double Click Point Cloud

    View full-size slide

  46. Getting DataStream from Kinect
    Int[] indices;
    Transformation trans; //Coordinate tranlation
    void Start() { /*omitted*/ }
    void InitKinect()
    {
    /*Omitted*/
    width = device.GetCalibration().
    depth_camera_calibration.resolution_width;
    height = device.GetCalibration().
    depth_camera_calibration.resolution_height;
    num = height * width;
    /*It’ll be used for transform coordinate system*/
    trans = device.GetCalibration().CreateTransformation();
    }

    View full-size slide

  47. Getting DataStream from Kinect
    using UnityEngine;
    using Microsoft.Azure.Kinect.Sensor;
    //Enable to use async function.
    using System.Threading.Tasks;
    public class KinectScript : MonoBehaviour
    {
    Device device;
    int width, height, num;
    Mesh mesh;
    Vector3[] vertices;
    Color32[] colors;
    int[] indices;
    /*Omitted*/

    View full-size slide

  48. Getting DataStream from Kinect
    void Start()
    {
    InitKinect();
    InitMesh();
    Task t = KinectLoop( );
    }
    private async Task KinectLoop( )
    {
    while (true)
    {
    //Continue to get DataStream from Kinect.
    }
    }

    View full-size slide

  49. Getting DataStream from Kinect
    While (true)
    {
    //Getting synchlonyzed frame of Kinect
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    //Getting Depth image with capture.Depth
    //And transform it into xyz by DeptuImageToPointCloud
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    //Getting vertices data.
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    /*See the next page*/
    }
    }

    View full-size slide

  50. Getting DataStream from Kinect
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    //Assigning coordinate of each vertex to vertices.
    for (int i = 0; i < num; i++)
    {
    vertices[i].x = pointCloud[i].X * 0.001f;
    vertices[i].y = pointCloud[i].Y * 0.001f;
    vertices[i].z = pointCloud[i].Z * 0.001f;
    }
    //set vertices to mesh.
    mesh.vertices = vertices;
    mesh.RecalculateBounds();
    }

    View full-size slide

  51. Modification of a Position of Camera
    ①MainCameraを選択
    ②Positionを0 0 0

    View full-size slide

  52. Run
    Click Scene Tab if you want to see
    point cloud from any view point
    Upside Down

    View full-size slide

  53. Modification of Y Coordinate
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    // Assigning coordinate of each vertex to vertices.
    for (int i = 0; i < num; i++)
    {
    vertices[i].x = pointCloud[i].X * 0.001f;
    vertices[i].y = -pointCloud[i].Y * 0.001f;
    vertices[i].z = pointCloud[i].Z * 0.001f;
    }
    // set vertices to mesh.
    mesh.vertices = vertices;
    mesh.RecalculateBounds();
    }

    View full-size slide

  54. Run
    Click Scene Tab if you want to see
    point cloud from any view point

    View full-size slide

  55. Assigning Color to Point Cloud
    while (true)
    {
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    //Getting color image which is matched to Depth resolution
    Image modifiedColor =
    trans.ColorImageToDepthCamera(capture);
    //Getting pixel array.
    BGRA[] colorArray =
    modifiedColor.GetPixels().ToArray();
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    /*See the next page*/
    KinectLoop Function

    View full-size slide

  56. Assigning Color to Point Cloud
    /*Continued from the previous page*/
    for (int i = 0; i < num; i++)
    {
    vertices[i].x = pointCloud[i].X * 0.001f;
    vertices[i].y = -pointCloud[i].Y * 0.001f;
    vertices[i].z = pointCloud[i].Z * 0.001f;
    colors[i].a = 255;
    colors [i].b = colorArray[i].B;
    colors [i].g = colorArray[i].G;
    colors [i].r = colorArray[i].R;
    }
    mesh.vertices = vertices;
    //Setting color to mesh.
    mesh.colors32 = colors;
    mesh.RecalculateBounds();

    View full-size slide

  57. AR Visualization with Vuforia
    File

    View full-size slide

  58. AR Visualization with Vuforia
    Build Settings

    View full-size slide

  59. AR Visualization with Vuforia
    Player Settings

    View full-size slide

  60. AR Visualization with Vuforia
    ①XR Settings
    ②Vuforia Augmented Reality

    View full-size slide

  61. AR Visualization with Vuforia
    Accept

    View full-size slide

  62. AR Visualization with Vuforia
    チェックされてればOK

    View full-size slide

  63. AR Visualization with Vuforia
    Delete MainCamera

    View full-size slide

  64. AR Visualization with Vuforia
    Right click

    View full-size slide

  65. AR Visualization with Vuforia
    ①Vuforia Engine
    ②ARCamera

    View full-size slide

  66. AR Visualization with Vuforia
    ①Rigght click
    ②Vuforia Engine
    ③Image

    View full-size slide

  67. AR Visualization with Vuforia
    ①Double ImageTarget

    View full-size slide

  68. AR Visualization with Vuforia
    Drag & Drop PointCloud
    Into ImageTarget

    View full-size slide

  69. AR Visualization with Vuforia
    Confirm PointCloud being
    child of ImageTarget

    View full-size slide

  70. Modification of Whole Size of Point Cloud
    ①PointCloud
    ②Set Position and Scale as below
    Position 0 0.5 -1
    Scale 1 1 1

    View full-size slide

  71. Modification of Whole Size of Point Cloud
    ②Open Vuforia Engine Configuration
    ①ARCamera

    View full-size slide

  72. Selecting Camera Device
    Select your Camera Device

    View full-size slide