Save 37% off PRO during our Black Friday Sale! »

Androidで
スクリーン配信をする技術

 Androidで
スクリーン配信をする技術

DroidKaigi 2020
https://droidkaigi.jp/2020/

Cb4d62dd7c1d325863696c45e9b6484d?s=128

Shumpei Urabe

January 23, 2020
Tweet

Transcript

  1. AndroidͰ
 εΫϦʔϯ഑৴Λ͢Δٕज़ 2020.01.23 ߹ಉձࣾφΫα
 ઎෦ ॡฏ

  2. ࣗݾ঺հ ઎෦ ॡฏ • ߹ಉձࣾφΫαɹ୅ද • ΤϯλϝͷϥΠϒ഑৴Λࢧ͍͑ͯ ·͢ • झຯ͸࡞ۂͱωοτϫʔΫӡ༻

    • Android։ൃͰҰ൪࢖͍ͬͯΔ΋ͷ ͸MediaCodecͱVulkan
  3. 1. MediaProjection AP I 2. MediaProjection Manager 3. VirtualDispla y

    4. MediaCodec 5. ө૾ϑΟϧλʔ 6. ը໘ճస 7. AudioPlaybackCapture ΋͘͡
  4. 1. MediaProjection AP I 2. MediaProjection Manager 3. VirtualDispla y

    4. MediaCodec 5. ө૾ϑΟϧλʔ 6. ը໘ճస 7. AudioPlaybackCapture ΋͘͡ BASIC HARD
  5. MediaProjection API 1.

  6. None
  7. None
  8. None
  9. None
  10. None
  11. MediaProjectionΛߏ੒͢Δཁૉ MediaProjection Manager VirtualDisplay SurfaceView MediaCodec AudioPlaybackCapture OpenGL ES

  12. MediaProjectionΛߏ੒͢Δཁૉ MediaProjection Manager VirtualDisplay SurfaceView MediaCodec AudioPlaybackCapture OpenGL ES it

    is Difficult.
  13. MediaProjectionΛߏ੒͢Δཁૉ MediaProjection Manager VirtualDisplay SurfaceView MediaCodec AudioPlaybackCapture OpenGL ES No,

    no.
 You can see this slide.
  14. MediaProjectionͷεςοϓ MediaProjection
 Manager Surface View createVirtualDisplay

  15. createMediaProjectionҎ߱ͷεςοϓ createVirtualDisplay MediaCodec RTMP etc. ImageRender Bitmap etc

  16. Security

  17. Security REQUEST_MEDIA_PROJECTION

  18. targetSdkVersion 29(Android 9) <?xml version="1.0" encoding="utf-8"? > <manifest xmlns:android="http://schemas.android.com/apk/res/android "

    package="your.package.name" > <uses-permission android:name="android.permission.FOREGROUND_SERVICE" / > <!-- লུ -- > </manifest>
  19. targetSdkVersion 29(Android 9) MediaProjection͸ForegroundͰผϓϩηεͰಈͨ͘Ίهड़ඞਢ <?xml version="1.0" encoding="utf-8"? > <manifest xmlns:android="http://schemas.android.com/apk/res/android

    " package="your.package.name" > <uses-permission android:name="android.permission.FOREGROUND_SERVICE" / > <!-- লུ -- > </manifest>
  20. Usecases ScreenShot Live Streaming

  21. MediaProjection Manager 2.

  22. Get Permission

  23. companion object companion object { private const val REQUEST_CAPTURE =

    1 private var projection: (MediaProjection?) -> Unit = { } val run: (context: Context, (MediaProjection?) -> Unit) -> Unit = { context, callback - > projection = callback context.startActivity(Intent(context, MediaProjectionModel::class.java).addFlags(Intent.FLAG_ACTIVITY_NEW_T ASK) ) } }
  24. getSystemService class MediaProjectionModel : Activity() { private lateinit var mediaProjectionManager:

    MediaProjectionManage r override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState ) mediaProjectionManager = getSystemService(Service.MEDIA_PROJECTION_SERVICE) as MediaProjectionManage r startActivityForResult(mediaProjectionManager.createScreenCaptureIntent (), REQUEST_CAPTURE ) }
  25. startActivityForResult override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {

    if (requestCode == REQUEST_CAPTURE && resultCode == RESULT_OK && data != null) { projection(mediaProjectionManager.getMediaProjection(resultCode , data) ) } finish( ) }
  26. Allow MediaProjection Service MediaProjection
 Manager Surface View createVirtualDisplay

  27. VirtualDisplay 3.

  28. Android View View SurfaceView TextureView

  29. Android View View SurfaceView TextureView ߴ଎ඳըʹ͸޲͍͍ͯͳ͍

  30. Android View View SurfaceView TextureView ߴ଎ඳըʹ͸޲͍͍ͯͳ͍ Android 7.0Ͱඇਪ঑

  31. Android View View SurfaceView TextureView ߴ଎ඳըʹ͸޲͍͍ͯͳ͍ Android 7.0Ͱඇਪ঑ High Perormance

  32. SurfaceView ௨ৗͷViewΫϥεΛܧঝͨ͠Ϋϥε UIεϨου͔Βಠཱͯ͠ॲཧͰ͖Δ GLSurfaceViewͰOpenGLΛ࢖ͬͨඳը΋Ͱ͖Δ

  33. VirtualDisplay View

  34. VirtualDisplay View VirtualDisplay Mirroring

  35. VirtualDisplay VirtualDisplay SurfaceView ImageRender Bitmapग़ྗ༻ͷSurfaceʹίϐʔ͢Δ ผͷSurfaceViewʹίϐʔ͢Δ

  36. ImageRender fun setupVirtualDisplay(): ImageReader? { val width = (metrics.widthPixels *

    scale).toInt() val height = (metrics.heightPixels * scale).toInt() val reader = ImageReader.newInstance(width , height, PixelFormat.RGBA_8888, 2).also { it.setOnImageAvailableListener(this, null) } virtualDisplay = mediaProjection?.createVirtualDisplay( "Capturing Display", width, height, metrics.densityDpi, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, reader!!.surface, null, null ) callback.changeState(StatesType.Running) return reader
  37. ImageRender override fun onImageAvailable(reader: ImageReader) { reader.acquireLatestImage().use { img ->

    runCatching { heepPlane = img?.planes?.get(0) ?: return@use null val width = heepPlane.rowStride / heepPlane.pixelStride val height = (metrics.heightPixels * scale).toInt() heepBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888).apply { copyPixelsFromBuffer(heepPlane.buffer) } callback.changeBitmap(heepBitmap) } } }
  38. Bitmap Rendering createVirtualDisplay MediaCodec RTMP etc. ImageRender Bitmap etc

  39. PixelFormat PixelFormat.RGBA_8888 PixelFormat.RGBX_8888

  40. VIRTUAL_DISPLAY_FLAG ఆ਺໊ આ໌ VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR ίϯςϯπΛϛϥʔϦϯάදࣔ͢Δ VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY ಠࣗͷίϯςϯπΛදࣔɻϛϥʔϦϯά͠ͳ͍ VIRTUAL_DISPLAY_FLAG_PRESENTATION ϓϨθϯςʔγϣϯϞʔυ VIRTUAL_DISPLAY_FLAG_PUBLIC

    HDMI΍WirelessσΟεϓϨΠ VIRTUAL_DISPLAY_FLAG_SECURE ҉߸Խରࡦ͕ࢪ͞ΕͨηΩϡΞͳσΟεϓϨΠ
  41. Allow MediaProjection Service MediaProjection
 Manager Surface View createVirtualDisplay

  42. MediaCodec 4.

  43. MediaCodec MediaFormatͰࢦఆ͞Εͨ΋ͷΛΤϯίʔυͨ͠Γ
 σίʔυͨ͠Γग़དྷΔAPI CallbackΛ࢖͏͜ͱͰඇಉظͰॲཧ͢Δ͜ͱ͕ग़དྷΔʢਪ঑ʣ

  44. MediaFormat val mime = MediaFormat.MIMETYPE_VIDEO_AVC val format = MediaFormat.createVideoFormat(mime, width,

    height) //ϑΥʔϚοτͷϓϩύςΟΛઃఆ //࠷௿ݶͷϓϩύςΟΛઃఆ͠ͳ͍ͱconfigureͰΤϥʔʹͳΔ format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) format.setInteger(MediaFormat.KEY_FRAME_RATE, 30) // LolipopͰ͸͜͜͸ແࢹ͞ΕΔ format.setInteger(MediaFormat.KEY_BIT_RATE, 1000 * 1024) format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 10) format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1000000L / 30)
  45. MediaFormat ϓϩύςΟ આ໌ KEY_COLOR_FORMAT ΧϥʔϑΥʔϚοτΛࢦఆ͠·͢ɻSurfaceͷ৔߹͸ωΠςΟϒRAWϑΥʔ ϚοτͰ͋Δ COLOR_FormatSurface Λࢦఆ͠·͢ɻ KEY_FRAME_RATE ϑϨʔϜϨʔτΛࢦఆ͠·͢ɻ60fpsͰΤϯίʔυ͢Δ͜ͱ΋ग़དྷ·͕͢ɺ

    ݹ͍୺຤ͩͱΧΫΧΫʹͳΔͷͰ30fps͋ͨΓ͕͓קΊͰ͢ɻ KEY_BIT_RATE ϏοτϨʔτΛࢦఆ͠·͢ɻMediaCodecͷH.264Τϯίʔμʔ͸࢒೦ͳ͕ Βѹॖޮ཰͕Α͘ͳ͍ͨΊɺߴΊʹઃఆ͠ͳ͍ͱϒϩοΫϊΠζ͕ग़΍͢ ͍Ͱ͢ɻ KEY_I_FRAME_INTERVAL ΩʔϑϨʔϜ(IϑϨʔϜ)ͷִؒΛઃఆ͠·͢ɻ௨ৗ͸10ϑϨʔϜຖͰ͍͍ͱ ࢥ͍·͕͢ɺΩʔϑϨʔϜִؒΛ୹͘͢Δͱ஗Ԇ͕গͳ͘ͳΓ·͢ɻ KEY_REPEAT_PREVIOUS_FRAME_AFTER SurfaceͰϑϨʔϜ͕͜ͳ͍৔߹ʹɺલͷϑϨʔϜΛ܁Γฦִؒ͢Λઃఆ͠ ·͢ɻ
  46. MediaCodec.Callback() codec = MediaCodec.createEncoderByType(mime) codec.setCallback(object: MediaCodec.Callback() { override fun onInputBufferAvailable(@NonNull

    codec: MediaCodec, index: Int) { Log.d("MediaCodec", "onInputBufferAvailable : " + codec.codecInfo) }
  47. MediaCodecList val codecName: String? = if (Build.VERSION_CODES.LOLLIPOP <= Build.VERSION.SDK_INT) {

    MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(mediaFormat) } else null return if (codecName != null) { MediaCodec.createByCodecName(codecName) } else { MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) }
  48. MediaCodec.Callback() override fun onOutputBufferAvailable(@NonNull codec: MediaCodec, index: Int, @NonNull info:

    MediaCodec.BufferInfo) { Log.d("MediaCodec", "onOutputBufferAvailable : $info") val buffer: ByteBuffer = codec.getOutputBuffer(index) val array = ByteArray(buffer.limit()) buffer.get(array) //Τϯίʔυ͞ΕͨσʔλΛૹ৴ Send(array) //όοϑΝΛղ์ codec.releaseOutputBuffer(index, false) }
  49. MediaCodec.Callback() override fun onError(@NonNull codec: MediaCodec, @NonNull e: CodecException) {

    Log.d("MediaCodec", "onError : " + e.message) } override fun onOutputFormatChanged(@NonNull codec: MediaCodec, @NonNull format: MediaFormat) { Log.d("MediaCodec", "onOutputFormatChanged : " + format.getString(MediaFormat.KEY_MIME)) } })
  50. Configure //ΤϯίʔμΛઃఆ codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) //ΤϯίʔμʹϑϨʔϜΛ౉͢ͷʹ࢖͏SurfaceΛऔಘ //configureͱstartͷؒͰݺͿඞཁ͋Γ inputSurface =

    codec.createInputSurface()
  51. VirtualDisplay VirtualDisplay SurfaceView ImageRender Bitmapग़ྗ༻ͷSurfaceʹίϐʔ͢Δ ผͷSurfaceViewʹίϐʔ͢Δ

  52. VirtualDisplay fun setupVirtualDisplay(inputSurface: Surface) { val width = (metrics.widthPixels *

    scale).toInt() val height = (metrics.heightPixels * scale).toInt() virtualDisplay = mediaProjection?.createVirtualDisplay( "Capturing Display", width, height, metrics.densityDpi, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, inputSurface, null, null ) callback.changeState(StatesType.Running) }
  53. MediaCodec Encode createVirtualDisplay MediaCodec RTMP etc. ImageRender Bitmap etc

  54. ө૾ϑΟϧλʔ 5.

  55. SurfaceView ௨ৗͷViewΫϥεΛܧঝͨ͠Ϋϥε UIεϨου͔Βಠཱͯ͠ॲཧͰ͖Δ GLSurfaceViewͰOpenGLΛ࢖ͬͨඳը΋Ͱ͖Δ

  56. VirtualDisplay View VirtualDisplay Mirroring

  57. VirtualDisplay View VirtualDisplay Mirroring Mirroring OpenGL ES 
 (Shader)

  58. GLSurfaceView SurfaceViewΑΓ΋ߴ଎ʹॲཧͰ͖Δ AndroidͰ͸EGLΛ࢖ͬͯOpenGLͱϒϦοδ͍ͯ͠Δ ShaderΛॻ͘͜ͱͰө૾ϑΟϧλʔΛ͚ͭΔ͜ͱ͕Ͱ͖Δ

  59. Grafika

  60. Grafika Google͕։ൃͨ͠άϥϑΟοΫεAPIͷαϯϓϧू Apache License 2.0Ͱެ։͍ͯ͠Δ

  61. setupEGL fun setupEgl(surface: Surface) { eglCore = EglCore() windowSurface =

    WindowSurface(eglCore, surface, true).apply { makeCurrent() } defaultTexture2dProgram = DefaultTexture2dProgram( DefaultTexture2dProgram.TextureTarget.TextureExt ) blurTexture2dProgram = GaussianBlurTexture2dProgram( outputWidth = outputSize.width, outputHeight = outputSize.height ) texId = GlUtil.createTextureObject(GLES11Ext.GL_TEXTURE_EXTERNAL_OES)
  62. setupEGL texId = GlUtil.createTextureObject(GLES11Ext.GL_TEXTURE_EXTERNAL_OES) GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat()) GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat())

    sprite2d = Sprite2d(Drawable2d(Drawable2d.Prefab.RECTANGLE)) .apply { setTexture(texId) setPosition(outputSize.width / 2f, outputSize.height / 2f) setScale(outputSize.width.toFloat(), outputSize.height.toFloat()) }
  63. setupEGL sourceSurfaceTexture = SurfaceTexture(texId).apply { setDefaultBufferSize(renderConfig.width, renderConfig.height) setOnFrameAvailableListener { synchronized(hasRenderFrame)

    { hasRenderFrame = true } try { updateTexImage() } catch (e: Exception) { Timber.e(e, "failed to updateTexImage.") } } } sourceSurface = Surface(sourceSurfaceTexture) }
  64. setupVirtualDisplay fun setupVirtualDisplay() { virtualDisplay = mediaProjection.createVirtualDisplay( "screen_capture", renderConfig.width, renderConfig.height,

    renderConfig.dpi, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, sourceSurface, null, null ) }
  65. ܨ͍͛ͯ͘ setupEgl(surface) setupVirtualDisplay() renderDisposable = Observable.interval( 0L, 1000L / renderConfig.frameRate,

    TimeUnit.MILLISECONDS ).observeOn(AndroidSchedulers.from(Looper.myLooper())) .subscribe({ drawFrame() }, { Timber.e("draw frame error") }) rotationChangeDetector.addListener(onRotationChange) notificationReceiveDetector.addListener(onNotificationReceive)
  66. ϑϨʔϜ͕དྷͨΒdraw͢Δ private fun drawFrame() { synchronized(hasRenderFrame) { if (!hasRenderFrame) return

    hasRenderFrame = false } windowSurface.makeCurrent() GLES20.glViewport(0, 0, outputSize.width, outputSize.height) if (blurStatus.shouldBlur()) { sprite2d.draw(blurTexture2dProgram, tmpMatrix) } else { sprite2d.draw(defaultTexture2dProgram, tmpMatrix) } windowSurface.swapBuffers() }
  67. ը໘ճస 6.

  68. ը໘ͷճసΛݕ஌͢Δ override fun onReceive(context: Context, intent: Intent) { val windowManager

    = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager val angle = when (windowManager.defaultDisplay.rotation) { Surface.ROTATION_0 -> 0 Surface.ROTATION_90 -> 90 Surface.ROTATION_180 -> 180 Surface.ROTATION_270 -> 270 else -> throw IllegalArgumentException("unknown rotation.") } listeners.forEach { it(angle) } }
  69. OpenGLͰॎԣൺΛ߹ΘͤΔ private fun updateRotation(rotationAngle: Int) { val diffAngle = rotationAngle

    - renderConfig.rotationAngle val (w, h) = if (Math.abs(diffAngle) == 270 || Math.abs(diffAngle) == 90) { val scale = if (outputSize.width > outputSize.height) { outputSize.width.toFloat() / outputSize.height } else { outputSize.height.toFloat() / outputSize.width } Pair(outputSize.width * scale, outputSize.height * scale) } else { Pair(outputSize.width.toFloat(), outputSize.height.toFloat()) } sprite2d.setScale(w, h) sprite2d.setRotationZ(diffAngle.toFloat()) }
  70. AudioPlaybackCapture 7.

  71. AudioPlaybackCapture Android 10ΑΓೖͬͨ୺຤ͷԻΛϧʔϓόοΫͰ͖Δػೳ ϞϊϥϧͰ͸ͳ͘εςϨΦͰग़ྗ͢Δ͜ͱ΋Մೳ MediaProjectionͷҰ෦ͱͯ͠ఏڙ͞Ε͍ͯΔ

  72. ྺ࢙ Android 5.0ͱڞʹొ৔ͨ͠MediaProjection ͦͷͱ͖ʹAudioRecordͰϧʔϓόοΫػೳ΋࣮૷͞Εͨ

  73. MediaRecorder.AudioSource REMOTE_SUBMIX ୺຤ͷԻ੠ΛϦϞʔτʹྲྀ͢
 ͨͩ͠Android 6.0ΑΓγεςϜݖݶ͕ඞཁʹͳͬͨ
 (࢖͑ͳ͘ͳͬͯ͠·ͬͨ)

  74. ౪ௌΞϓϦͷྲྀߦ REMOTE_SUBMIX͸௨࿩Ի΋ΩϟϓνϟՄೳͰ͋ͬͨ ͦͷͨΊɺGoogle͸Android 6.0Ͱར༻ෆՄʹͨ͠ ͔͠͠ɺAndroid 10ͰAudioPlaybackCaptureͱͯ͠෮׆

  75. ੍໿ • compileSdkVersion 29 ͷΞϓϦ͸σϑΥϧτͰΩϟϓνϟʔ͞ΕΔɻແޮԽ ͍ͨ͠৔߹͸AndroidManifest ʹ android:allowAudioPlaybackCapture="false" ͕ඞཁ •

    compileSdkVersion 28 ҎલͷΞϓϦ͸σϑΥϧτͰΩϟϓνϟ͕ϒϩοΫ͞ ΕΔɻΩϟϓνϟ͢Δ৔߹͸ android:allowAudioPlaybackCapture="true" Λ ManifestʹೖΕΔඞཁ͕͋Δ
  76. AudioPlaybackCaptureConfiguration public boolean prepareInternalAudio(int bitrate, int sampleRate, boolean isStereo) {

    if (mediaProjection == null) { mediaProjection = mediaProjectionManager.getMediaProjection(resultCode, data); } AudioPlaybackCaptureConfiguration config = new AudioPlaybackCaptureConfiguration .Builder(mediaProjection) .addMatchingUsage(AudioAttributes.USAGE_MEDIA) .addMatchingUsage(AudioAttributes.USAGE_GAME) .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN) .build(); microphoneManager.createInternalMicrophone(config, sampleRate, isStereo); prepareAudioRtp(isStereo, sampleRate); return audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, microphoneManager.getMaxInputSize()); }
  77. AudioPlaybackCaptureConfiguration public void createInternalMicrophone(AudioPlaybackCaptureConfiguration config, int sampleRate, boolean isStereo) {

    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { this.sampleRate = sampleRate; if (!isStereo) channel = AudioFormat.CHANNEL_IN_MONO; audioRecord = new AudioRecord.Builder() .setAudioPlaybackCaptureConfig(config) .setAudioFormat(new AudioFormat.Builder() .setEncoding(audioFormat) .setSampleRate(sampleRate) .setChannelMask(channel) .build()) .setBufferSizeInBytes(getPcmBufferSize()) .build();
  78. ·ͱΊ

  79. εΫϦʔϯ഑৴Λ͢Δٕज़ ಈը഑৴͸༷ʑͳٕज़ͷ૊Έ߹Θٕͤ ݱࡏͰ͸ଟ͘ͷϥΠϒϥϦʹΑΓɺ؆୯ʹ࣮૷͕Ͱ͖Δ ੋඇ͋ͱͰνϟϨϯδͯ͠Έ͍ͯͩ͘͞ʂ

  80. Thank you.