CameraX × ML KitでパスポートOCR機能を実装

DroidKaig2022で「CameraX × ML KitでパスポートOCR機能を実装」というテーマで発表をさせていただきました。その資料と動画と、セッション内で紹介しているサンプルコードを共有します。

資料

動画

https://www.youtube.com/watch?v=SBu2J8blfac

サンプルコード

プレビューの配置

is PermissionStatus.Granted -> {
    val context = LocalContext.current
    val cameraProviderFuture = remember { ProcessCameraProvider.getInstance(context) }

    AndroidView(
	modifier = Modifier.fillMaxSize().padding(it),
	factory = { ctx ->
	    PreviewView(ctx).apply {
		setupCamera(
		    previewView = this,
		    lifecycleOwner = lifecycleOwner,
		    cameraProviderFuture = cameraProviderFuture,
		    onPassportRecognized = {
			// 認識したTextをViewModel等に渡して処理
		    }
		)
	    }
	},
    )
}

カメラのセットアップ

private fun setupCamera(
    previewView: PreviewView,
    lifecycleOwner: LifecycleOwner,
    cameraProviderFuture: ListenableFuture<ProcessCameraProvider>,
    onPassportRecognized: (mrzText: String) -> Unit,
) {
    cameraProviderFuture.addListener({
        try {
            val cameraProvider = cameraProviderFuture.get()

            val preview = Preview.Builder().build().apply {
                setSurfaceProvider(previewView.surfaceProvider)
            }

            val cameraSelector =
                CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build()

            val imageAnalysis = ImageAnalysis.Builder()
                .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST).build()
                .apply {
                    setAnalyzer(
                        Executors.newSingleThreadExecutor(),
                        PassportAnalyzer(onPassportRecognized)
                    )
                }

            val useCaseGroup = UseCaseGroup.Builder()
                .addUseCase(preview)
                .addUseCase(imageAnalysis)
                .build()

            cameraProvider.unbindAll()
            cameraProvider.bindToLifecycle(
                lifecycleOwner, cameraSelector, useCaseGroup
            )
        } catch (error: Throwable) {
            // ...
        }
    }, ContextCompat.getMainExecutor(previewView.context))
}

画像解析

@androidx.annotation.OptIn(androidx.camera.core.ExperimentalGetImage::class)
class PassportAnalyzer(
    private val onRecognized: (mrzText: String) -> Unit
) : ImageAnalysis.Analyzer {

    private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)

    override fun analyze(imageProxy: ImageProxy) {
        val image = imageProxy.image

        if (image != null) {
            val imageRotation = imageProxy.imageInfo.rotationDegrees
            val inputImage = InputImage.fromMediaImage(image, imageRotation)
            recognizer.process(inputImage)
                .addOnSuccessListener { recognizedText ->
                    val textBlocks = recognizedText.textBlocks
                    extractMrzText(textBlocks)?.let { onRecognized(it) }
                    imageProxy.close()
                }.addOnFailureListener {
                    imageProxy.close()
                }
        } else {
            imageProxy.close()
        }
    }
}

デバイス回転時の処理

val orientationEventListener = object : OrientationEventListener(context) {
    override fun onOrientationChanged(orientation : Int) {
        val rotation : Int = when (orientation) {
            in 45..134 -> Surface.ROTATION_270
            in 135..224 -> Surface.ROTATION_180
            in 225..314 -> Surface.ROTATION_90
            else -> Surface.ROTATION_0
        }
        imageAnalysis.targetRotation = rotation
    }
}

MRZの抽出

private fun extractMrzText(textBlocks: List<Text.TextBlock>): String? {
    val textList = textBlocks.map { it.text }
    val joinedText = textList.joinToString().replace("\\s".toRegex(), "")
    val index = joinedText.indexOf("P<")
    val reg = Regex(".*P<.*")
    val mrz = if (reg.containsMatchIn(joinedText) && joinedText.length >= index + 88) {
        joinedText.substring(index, index + 88)
    } else {
        null
    }
    mrz?.let { Timber.d("ocr mrz $mrz") }
    return mrz
}