Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Getting Started with Azure Kinect DK

TakashiYoshinaga
September 08, 2019

Getting Started with Azure Kinect DK

Slice deck of tutorial of contents creation with Azure Kinect and Unity.

TakashiYoshinaga

September 08, 2019
Tweet

More Decks by TakashiYoshinaga

Other Decks in Technology

Transcript

  1. Getting Started with
    Azure Kinect

    View Slide

  2. Download Pre-Build SDK and Asset
    http://arfukuoka.lolipop.jp/Azure
    Kinect/Sample.zip

    View Slide

  3. Goal of the Tutorial
    Visualization of Colored Point Cloud on AR marker

    View Slide

  4. Creating Project
    New

    View Slide

  5. Creating Project
    Create Project
    Project Name
    Directory

    View Slide

  6. Switching .NET Environment
    ①File
    ②Build Setting

    View Slide

  7. Switching .NET Environment
    Player Settings

    View Slide

  8. Switching .NET Environment
    Other Settings

    View Slide

  9. Switching .NET Environment
    Scripting Runtime Versionを
    .NET 4.X Equivalent
    Api Compatibility Levelを
    .NET 4.X

    View Slide

  10. Installation of Azure Kinect SDK
    Right ClickAssets

    View Slide

  11. Installation of Azure Kinect SDK
    ①Create
    ②Folder

    View Slide

  12. Installation of Azure Kinect SDK
    Rename it to Plugins

    View Slide

  13. Installation of Azure Kinect SDK
    Click Plugins

    View Slide

  14. Installation of Azure Kinect SDK
    ①Open following directory by explorer
    Sample¥Plugins
    ②Drag & Drop all Files
    into Plugins of this project

    View Slide

  15. Installation of Azure Kinect SDK
    ①Open the directory of which Azure Kinect SDK
    is installed
    C:¥Program Files¥Azure Kinect SDK
    v1.2.0¥sdk¥windows-desktop¥amd64¥release¥bin
    ②Drag and drop depthengine_2_0.dll
    into Plugins of your project

    View Slide

  16. Adding Object to Visualize Point Cloud
    Right Click

    View Slide

  17. Adding Object to Visualize Point Cloud
    Create Empty

    View Slide

  18. Adding Object to Visualize Point Cloud
    ①GameObject
    ②Rename into PointCloud

    View Slide

  19. Adding Object to Visualize Point Cloud
    Add Component

    View Slide

  20. Adding Object to Visualize Point Cloud
    ①seach ”mesh”
    ②Mesh Filter

    View Slide

  21. Adding Object to Visualize Point Cloud
    Add Component

    View Slide

  22. Adding Object to Visualize Point Cloud
    Mesh Renderer

    View Slide

  23. Adding Object to Visualize Point Cloud
    Open Materials

    View Slide

  24. Adding Object to Visualize Point Cloud
    Next step, material for drawing
    point cloud will be set to Element0

    View Slide

  25. Adding a Material to Draw Point Cloud
    ①Sample Folder
    ②Double Click PointCloud
    ③Import

    View Slide

  26. Adding a Material to Draw Point Cloud
    ①Assets
    ②Check ColoredVertex & PointCloud are added

    View Slide

  27. Adding a Material to Draw Point Cloud
    ①PointCloud
    ②Drag & Drop PointCloud
    into Element0

    View Slide

  28. Adding a Material to Draw Point Cloud
    Custom/ColoredVertex

    View Slide

  29. Adding a Script to Draw Point Cloud
    ①Point Cloud
    ②Add Component

    View Slide

  30. Adding a Script to Draw Point Cloud
    ①Remove search word
    ②New Script
    ③KinectScript
    ④Create and Add

    View Slide

  31. Adding a Script to Draw Point Cloud
    Check Script is added

    View Slide

  32. Ctrl +S

    View Slide

  33. Adding a Script to Draw Point Cloud
    Double Click KinectScript
    to open this script

    View Slide

  34. Adding a Script to Draw Point Cloud

    View Slide

  35. Start and Stop Azure Kinect
    using UnityEngine;
    using Microsoft.Azure.Kinect.Sensor;
    public class KinectScript : MonoBehaviour
    {
    Device device; //Variable to access to Kinect
    void Start()
    {
    InitKinect(); //Initialization of Kinect
    }
    void InitKinect()
    {
    //See Next Page.
    }
    /*Omitted */
    }

    View Slide

  36. Adding a Script to Draw Point Cloud
    void InitKinect()
    {
    device = Device.Open(0); //OpenKinect
    device.StartCameras(new DeviceConfiguration
    {
    ColorFormat = ImageFormat.ColorBGRA32,
    ColorResolution = ColorResolution.R720p,
    DepthMode = DepthMode.NFOV_Unbinned,
    SynchronizedImagesOnly = true,
    CameraFPS = FPS.FPS30,
    });
    }
    void OnDestroy() //Stop Kinect
    {
    device.StopCameras();
    }

    View Slide

  37. Run
    Play

    View Slide

  38. Run
    LED is turned on

    View Slide

  39. Run
    Stop Play again

    View Slide

  40. Preparation of Drawing Point Cloud
    Device device;
    int width, height, num; //width, height and total Pixels
    void Start(){ /*Omitted*/ }
    void InitKinect()
    {
    device = Device.Open(0);
    device.StartCameras ( /*Omitted*/ );
    //Calculation of total number of pixel.
    width = device.GetCalibration().
    depth_camera_calibration.resolution_width;
    height = device.GetCalibration().
    depth_camera_calibration.resolution_height;
    num = height * width;
    }

    View Slide

  41. Preparation of Drawing Point Cloud
    int width, height, num;
    Mesh mesh; //Used for Drawing Shape
    Vector3[] vertices; //Position of each point.
    Color32[] colors; //Color of each point.
    int[] indices; //Index list of drawing point
    void Start()
    {
    InitKinect();
    InitMesh(); //Initialization of mesh
    }
    void InitMesh()
    {
    //See the next page.
    }

    View Slide

  42. Preparation of Drawing Point Cloud
    void InitMesh()
    {
    mesh = new Mesh();
    //Enable drawing more than 65535 points
    mesh.indexFormat =
    UnityEngine.Rendering.IndexFormat.UInt32;
    //allocation of memory for treating information to
    //draw point cloud.
    vertices = new Vector3[num];
    colors = new Color32[num];
    indices = new int[num];
    //See next page
    }

    View Slide

  43. Next Step
    Draw points as many number as pixel of depth image.

    View Slide

  44. Drawing Points
    /*Continued from previous step (in InitMesh function)*/
    int index = 0; //index of vertex array.
    for (int y = 0; y < height; y++)
    {
    for (int x = 0; x < width; x++)
    {
    vertices[index].x = x ;
    vertices[index].y = y ;
    vertices[index].z = 2;
    colors[index].r = 0;
    colors [index].g = 0;
    colors [index].b = 255;
    colors [index].a = 255;
    indices[index] = index;
    index++;
    }
    }
    Temporally setting the
    coordinates of each vertex
    Temporally setting the
    color of each vertex
    Setting drawing list.

    View Slide

  45. Drawing Points
    /*Continued from the previous page.*/
    int index = 0;
    for (int y = 0; y < height; y++)
    {
    for (int x = 0; x < width; x++)
    {
    /*Omitted*/
    }
    }
    //Assignment vertices and colors to mesh.
    mesh.vertices = vertices;
    mesh.colors32 = colors;
    //Setting drawing list and mesh topology.
    mesh.SetIndices(indices, MeshTopology.Points, 0);
    //Assingnment of mesh to MeshFilter of the gameObject.
    gameObject.GetComponent().mesh = mesh;

    View Slide

  46. Run
    Camera is too close to points

    View Slide

  47. Run
    ①Scene Tab
    ②Double Click Point Cloud

    View Slide

  48. Getting DataStream from Kinect
    Int[] indices;
    Transformation trans; //Coordinate tranlation
    void Start() { /*omitted*/ }
    void InitKinect()
    {
    /*Omitted*/
    width = device.GetCalibration().
    depth_camera_calibration.resolution_width;
    height = device.GetCalibration().
    depth_camera_calibration.resolution_height;
    num = height * width;
    /*It’ll be used for transform coordinate system*/
    trans = device.GetCalibration().CreateTransformation();
    }

    View Slide

  49. Getting DataStream from Kinect
    using UnityEngine;
    using Microsoft.Azure.Kinect.Sensor;
    //Enable to use async function.
    using System.Threading.Tasks;
    public class KinectScript : MonoBehaviour
    {
    Device device;
    int width, height, num;
    Mesh mesh;
    Vector3[] vertices;
    Color32[] colors;
    int[] indices;
    /*Omitted*/

    View Slide

  50. Getting DataStream from Kinect
    void Start()
    {
    InitKinect();
    InitMesh();
    Task t = KinectLoop( );
    }
    private async Task KinectLoop( )
    {
    while (true)
    {
    //Continue to get DataStream from Kinect.
    }
    }

    View Slide

  51. Getting DataStream from Kinect
    While (true)
    {
    //Getting synchlonyzed frame of Kinect
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    //Getting Depth image with capture.Depth
    //And transform it into xyz by DeptuImageToPointCloud
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    //Getting vertices data.
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    /*See the next page*/
    }
    }

    View Slide

  52. Getting DataStream from Kinect
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    //Assigning coordinate of each vertex to vertices.
    for (int i = 0; i < num; i++)
    {
    vertices[i].x = pointCloud[i].X * 0.001f;
    vertices[i].y = pointCloud[i].Y * 0.001f;
    vertices[i].z = pointCloud[i].Z * 0.001f;
    }
    //set vertices to mesh.
    mesh.vertices = vertices;
    mesh.RecalculateBounds();
    }

    View Slide

  53. Modification of a Position of Camera
    ①MainCameraを選択
    ②Positionを0 0 0

    View Slide

  54. Run
    Click Scene Tab if you want to see
    point cloud from any view point
    Upside Down

    View Slide

  55. Modification of Y Coordinate
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    // Assigning coordinate of each vertex to vertices.
    for (int i = 0; i < num; i++)
    {
    vertices[i].x = pointCloud[i].X * 0.001f;
    vertices[i].y = -pointCloud[i].Y * 0.001f;
    vertices[i].z = pointCloud[i].Z * 0.001f;
    }
    // set vertices to mesh.
    mesh.vertices = vertices;
    mesh.RecalculateBounds();
    }

    View Slide

  56. Run
    Click Scene Tab if you want to see
    point cloud from any view point

    View Slide

  57. Assigning Color to Point Cloud
    while (true)
    {
    using (Capture capture = await Task.Run(() =>
    device.GetCapture()).ConfigureAwait(true))
    {
    //Getting color image which is matched to Depth resolution
    Image modifiedColor =
    trans.ColorImageToDepthCamera(capture);
    //Getting pixel array.
    BGRA[] colorArray =
    modifiedColor.GetPixels().ToArray();
    Image pImage =
    trans.DepthImageToPointCloud(capture.Depth);
    Short3[] pointCloud =
    pImage.GetPixels().ToArray();
    /*See the next page*/
    KinectLoop Function

    View Slide

  58. Assigning Color to Point Cloud
    /*Continued from the previous page*/
    for (int i = 0; i < num; i++)
    {
    vertices[i].x = pointCloud[i].X * 0.001f;
    vertices[i].y = -pointCloud[i].Y * 0.001f;
    vertices[i].z = pointCloud[i].Z * 0.001f;
    colors[i].a = 255;
    colors [i].b = colorArray[i].B;
    colors [i].g = colorArray[i].G;
    colors [i].r = colorArray[i].R;
    }
    mesh.vertices = vertices;
    //Setting color to mesh.
    mesh.colors32 = colors;
    mesh.RecalculateBounds();

    View Slide

  59. Run

    View Slide

  60. AR Visualization with Vuforia
    File

    View Slide

  61. AR Visualization with Vuforia
    Build Settings

    View Slide

  62. AR Visualization with Vuforia
    Player Settings

    View Slide

  63. AR Visualization with Vuforia
    ①XR Settings
    ②Vuforia Augmented Reality

    View Slide

  64. AR Visualization with Vuforia
    Accept

    View Slide

  65. AR Visualization with Vuforia
    チェックされてればOK

    View Slide

  66. AR Visualization with Vuforia
    Delete MainCamera

    View Slide

  67. AR Visualization with Vuforia
    Right click

    View Slide

  68. AR Visualization with Vuforia
    ①Vuforia Engine
    ②ARCamera

    View Slide

  69. AR Visualization with Vuforia
    ①Rigght click
    ②Vuforia Engine
    ③Image

    View Slide

  70. AR Visualization with Vuforia
    ①Double ImageTarget

    View Slide

  71. AR Visualization with Vuforia
    Drag & Drop PointCloud
    Into ImageTarget

    View Slide

  72. AR Visualization with Vuforia
    Confirm PointCloud being
    child of ImageTarget

    View Slide

  73. Modification of Whole Size of Point Cloud
    ①PointCloud
    ②Set Position and Scale as below
    Position 0 0.5 -1
    Scale 1 1 1

    View Slide

  74. Modification of Whole Size of Point Cloud
    ②Open Vuforia Engine Configuration
    ①ARCamera

    View Slide

  75. Selecting Camera Device
    Select your Camera Device

    View Slide

  76. Marker

    View Slide

  77. Completion

    View Slide