Slide 1
Slide 1 text
Getting Started with
Azure Kinect
Slide 2
Slide 2 text
Download Pre-Build SDK and Asset
http://arfukuoka.lolipop.jp/Azure
Kinect/Sample.zip
Slide 3
Slide 3 text
Goal of the Tutorial
Visualization of Colored Point Cloud on AR marker
Slide 4
Slide 4 text
Creating Project
New
Slide 5
Slide 5 text
Creating Project
Create Project
Project Name
Directory
Slide 6
Slide 6 text
Switching .NET Environment
①File
②Build Setting
Slide 7
Slide 7 text
Switching .NET Environment
Player Settings
Slide 8
Slide 8 text
Switching .NET Environment
Other Settings
Slide 9
Slide 9 text
Switching .NET Environment
Scripting Runtime Versionを
.NET 4.X Equivalent
Api Compatibility Levelを
.NET 4.X
Slide 10
Slide 10 text
Installation of Azure Kinect SDK
Right ClickAssets
Slide 11
Slide 11 text
Installation of Azure Kinect SDK
①Create
②Folder
Slide 12
Slide 12 text
Installation of Azure Kinect SDK
Rename it to Plugins
Slide 13
Slide 13 text
Installation of Azure Kinect SDK
Click Plugins
Slide 14
Slide 14 text
Installation of Azure Kinect SDK
①Open following directory by explorer
Sample¥Plugins
②Drag & Drop all Files
into Plugins of this project
Slide 15
Slide 15 text
Installation of Azure Kinect SDK
①Open the directory of which Azure Kinect SDK
is installed
C:¥Program Files¥Azure Kinect SDK
v1.2.0¥sdk¥windows-desktop¥amd64¥release¥bin
②Drag and drop depthengine_2_0.dll
into Plugins of your project
Slide 16
Slide 16 text
Adding Object to Visualize Point Cloud
Right Click
Slide 17
Slide 17 text
Adding Object to Visualize Point Cloud
Create Empty
Slide 18
Slide 18 text
Adding Object to Visualize Point Cloud
①GameObject
②Rename into PointCloud
Slide 19
Slide 19 text
Adding Object to Visualize Point Cloud
Add Component
Slide 20
Slide 20 text
Adding Object to Visualize Point Cloud
①seach ”mesh”
②Mesh Filter
Slide 21
Slide 21 text
Adding Object to Visualize Point Cloud
Add Component
Slide 22
Slide 22 text
Adding Object to Visualize Point Cloud
Mesh Renderer
Slide 23
Slide 23 text
Adding Object to Visualize Point Cloud
Open Materials
Slide 24
Slide 24 text
Adding Object to Visualize Point Cloud
Next step, material for drawing
point cloud will be set to Element0
Slide 25
Slide 25 text
Adding a Material to Draw Point Cloud
①Sample Folder
②Double Click PointCloud
③Import
Slide 26
Slide 26 text
Adding a Material to Draw Point Cloud
①Assets
②Check ColoredVertex & PointCloud are added
Slide 27
Slide 27 text
Adding a Material to Draw Point Cloud
①PointCloud
②Drag & Drop PointCloud
into Element0
Slide 28
Slide 28 text
Adding a Material to Draw Point Cloud
Custom/ColoredVertex
Slide 29
Slide 29 text
Adding a Script to Draw Point Cloud
①Point Cloud
②Add Component
Slide 30
Slide 30 text
Adding a Script to Draw Point Cloud
①Remove search word
②New Script
③KinectScript
④Create and Add
Slide 31
Slide 31 text
Adding a Script to Draw Point Cloud
Check Script is added
Slide 32
Slide 32 text
Ctrl +S
Slide 33
Slide 33 text
Adding a Script to Draw Point Cloud
Double Click KinectScript
to open this script
Slide 34
Slide 34 text
Adding a Script to Draw Point Cloud
Slide 35
Slide 35 text
Start and Stop Azure Kinect
using UnityEngine;
using Microsoft.Azure.Kinect.Sensor;
public class KinectScript : MonoBehaviour
{
Device device; //Variable to access to Kinect
void Start()
{
InitKinect(); //Initialization of Kinect
}
void InitKinect()
{
//See Next Page.
}
/*Omitted */
}
Slide 36
Slide 36 text
Adding a Script to Draw Point Cloud
void InitKinect()
{
device = Device.Open(0); //OpenKinect
device.StartCameras(new DeviceConfiguration
{
ColorFormat = ImageFormat.ColorBGRA32,
ColorResolution = ColorResolution.R720p,
DepthMode = DepthMode.NFOV_Unbinned,
SynchronizedImagesOnly = true,
CameraFPS = FPS.FPS30,
});
}
void OnDestroy() //Stop Kinect
{
device.StopCameras();
}
Slide 37
Slide 37 text
Run
Play
Slide 38
Slide 38 text
Run
LED is turned on
Slide 39
Slide 39 text
Run
Stop Play again
Slide 40
Slide 40 text
Preparation of Drawing Point Cloud
Device device;
int width, height, num; //width, height and total Pixels
void Start(){ /*Omitted*/ }
void InitKinect()
{
device = Device.Open(0);
device.StartCameras ( /*Omitted*/ );
//Calculation of total number of pixel.
width = device.GetCalibration().
depth_camera_calibration.resolution_width;
height = device.GetCalibration().
depth_camera_calibration.resolution_height;
num = height * width;
}
Slide 41
Slide 41 text
Preparation of Drawing Point Cloud
int width, height, num;
Mesh mesh; //Used for Drawing Shape
Vector3[] vertices; //Position of each point.
Color32[] colors; //Color of each point.
int[] indices; //Index list of drawing point
void Start()
{
InitKinect();
InitMesh(); //Initialization of mesh
}
void InitMesh()
{
//See the next page.
}
Slide 42
Slide 42 text
Preparation of Drawing Point Cloud
void InitMesh()
{
mesh = new Mesh();
//Enable drawing more than 65535 points
mesh.indexFormat =
UnityEngine.Rendering.IndexFormat.UInt32;
//allocation of memory for treating information to
//draw point cloud.
vertices = new Vector3[num];
colors = new Color32[num];
indices = new int[num];
//See next page
}
Slide 43
Slide 43 text
Next Step
Draw points as many number as pixel of depth image.
Slide 44
Slide 44 text
Drawing Points
/*Continued from previous step (in InitMesh function)*/
int index = 0; //index of vertex array.
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
vertices[index].x = x ;
vertices[index].y = y ;
vertices[index].z = 2;
colors[index].r = 0;
colors [index].g = 0;
colors [index].b = 255;
colors [index].a = 255;
indices[index] = index;
index++;
}
}
Temporally setting the
coordinates of each vertex
Temporally setting the
color of each vertex
Setting drawing list.
Slide 45
Slide 45 text
Drawing Points
/*Continued from the previous page.*/
int index = 0;
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
/*Omitted*/
}
}
//Assignment vertices and colors to mesh.
mesh.vertices = vertices;
mesh.colors32 = colors;
//Setting drawing list and mesh topology.
mesh.SetIndices(indices, MeshTopology.Points, 0);
//Assingnment of mesh to MeshFilter of the gameObject.
gameObject.GetComponent().mesh = mesh;
Slide 46
Slide 46 text
Run
Camera is too close to points
Slide 47
Slide 47 text
Run
①Scene Tab
②Double Click Point Cloud
Slide 48
Slide 48 text
Getting DataStream from Kinect
Int[] indices;
Transformation trans; //Coordinate tranlation
void Start() { /*omitted*/ }
void InitKinect()
{
/*Omitted*/
width = device.GetCalibration().
depth_camera_calibration.resolution_width;
height = device.GetCalibration().
depth_camera_calibration.resolution_height;
num = height * width;
/*It’ll be used for transform coordinate system*/
trans = device.GetCalibration().CreateTransformation();
}
Slide 49
Slide 49 text
Getting DataStream from Kinect
using UnityEngine;
using Microsoft.Azure.Kinect.Sensor;
//Enable to use async function.
using System.Threading.Tasks;
public class KinectScript : MonoBehaviour
{
Device device;
int width, height, num;
Mesh mesh;
Vector3[] vertices;
Color32[] colors;
int[] indices;
/*Omitted*/
Slide 50
Slide 50 text
Getting DataStream from Kinect
void Start()
{
InitKinect();
InitMesh();
Task t = KinectLoop( );
}
private async Task KinectLoop( )
{
while (true)
{
//Continue to get DataStream from Kinect.
}
}
Slide 51
Slide 51 text
Getting DataStream from Kinect
While (true)
{
//Getting synchlonyzed frame of Kinect
using (Capture capture = await Task.Run(() =>
device.GetCapture()).ConfigureAwait(true))
{
//Getting Depth image with capture.Depth
//And transform it into xyz by DeptuImageToPointCloud
Image pImage =
trans.DepthImageToPointCloud(capture.Depth);
//Getting vertices data.
Short3[] pointCloud =
pImage.GetPixels().ToArray();
/*See the next page*/
}
}
Slide 52
Slide 52 text
Getting DataStream from Kinect
using (Capture capture = await Task.Run(() =>
device.GetCapture()).ConfigureAwait(true))
{
Image pImage =
trans.DepthImageToPointCloud(capture.Depth);
Short3[] pointCloud =
pImage.GetPixels().ToArray();
//Assigning coordinate of each vertex to vertices.
for (int i = 0; i < num; i++)
{
vertices[i].x = pointCloud[i].X * 0.001f;
vertices[i].y = pointCloud[i].Y * 0.001f;
vertices[i].z = pointCloud[i].Z * 0.001f;
}
//set vertices to mesh.
mesh.vertices = vertices;
mesh.RecalculateBounds();
}
Slide 53
Slide 53 text
Modification of a Position of Camera
①MainCameraを選択
②Positionを0 0 0
Slide 54
Slide 54 text
Run
Click Scene Tab if you want to see
point cloud from any view point
Upside Down
Slide 55
Slide 55 text
Modification of Y Coordinate
using (Capture capture = await Task.Run(() =>
device.GetCapture()).ConfigureAwait(true))
{
Image pImage =
trans.DepthImageToPointCloud(capture.Depth);
Short3[] pointCloud =
pImage.GetPixels().ToArray();
// Assigning coordinate of each vertex to vertices.
for (int i = 0; i < num; i++)
{
vertices[i].x = pointCloud[i].X * 0.001f;
vertices[i].y = -pointCloud[i].Y * 0.001f;
vertices[i].z = pointCloud[i].Z * 0.001f;
}
// set vertices to mesh.
mesh.vertices = vertices;
mesh.RecalculateBounds();
}
Slide 56
Slide 56 text
Run
Click Scene Tab if you want to see
point cloud from any view point
Slide 57
Slide 57 text
Assigning Color to Point Cloud
while (true)
{
using (Capture capture = await Task.Run(() =>
device.GetCapture()).ConfigureAwait(true))
{
//Getting color image which is matched to Depth resolution
Image modifiedColor =
trans.ColorImageToDepthCamera(capture);
//Getting pixel array.
BGRA[] colorArray =
modifiedColor.GetPixels().ToArray();
Image pImage =
trans.DepthImageToPointCloud(capture.Depth);
Short3[] pointCloud =
pImage.GetPixels().ToArray();
/*See the next page*/
KinectLoop Function
Slide 58
Slide 58 text
Assigning Color to Point Cloud
/*Continued from the previous page*/
for (int i = 0; i < num; i++)
{
vertices[i].x = pointCloud[i].X * 0.001f;
vertices[i].y = -pointCloud[i].Y * 0.001f;
vertices[i].z = pointCloud[i].Z * 0.001f;
colors[i].a = 255;
colors [i].b = colorArray[i].B;
colors [i].g = colorArray[i].G;
colors [i].r = colorArray[i].R;
}
mesh.vertices = vertices;
//Setting color to mesh.
mesh.colors32 = colors;
mesh.RecalculateBounds();
Slide 59
Slide 59 text
Run
Slide 60
Slide 60 text
AR Visualization with Vuforia
File
Slide 61
Slide 61 text
AR Visualization with Vuforia
Build Settings
Slide 62
Slide 62 text
AR Visualization with Vuforia
Player Settings
Slide 63
Slide 63 text
AR Visualization with Vuforia
①XR Settings
②Vuforia Augmented Reality
Slide 64
Slide 64 text
AR Visualization with Vuforia
Accept
Slide 65
Slide 65 text
AR Visualization with Vuforia
チェックされてればOK
Slide 66
Slide 66 text
AR Visualization with Vuforia
Delete MainCamera
Slide 67
Slide 67 text
AR Visualization with Vuforia
Right click
Slide 68
Slide 68 text
AR Visualization with Vuforia
①Vuforia Engine
②ARCamera
Slide 69
Slide 69 text
AR Visualization with Vuforia
①Rigght click
②Vuforia Engine
③Image
Slide 70
Slide 70 text
AR Visualization with Vuforia
①Double ImageTarget
Slide 71
Slide 71 text
AR Visualization with Vuforia
Drag & Drop PointCloud
Into ImageTarget
Slide 72
Slide 72 text
AR Visualization with Vuforia
Confirm PointCloud being
child of ImageTarget
Slide 73
Slide 73 text
Modification of Whole Size of Point Cloud
①PointCloud
②Set Position and Scale as below
Position 0 0.5 -1
Scale 1 1 1
Slide 74
Slide 74 text
Modification of Whole Size of Point Cloud
②Open Vuforia Engine Configuration
①ARCamera
Slide 75
Slide 75 text
Selecting Camera Device
Select your Camera Device
Slide 76
Slide 76 text
Marker
Slide 77
Slide 77 text
Completion