简介

本教程将介绍如何使用OpenCV深度学习模块在Android设备上运行深度学习网络。本教程针对Android Studio 2022.2.1编写。

需求

从 https://developer.android.com.cn/studio 下载并安装Android Studio。
从 https://github.com/opencv/opencv/releases 获取最新的预构建Android版OpenCV发行版并解压（例如，opencv-4.X.Y-android-sdk.zip，最低版本要求为4.9）。
从 https://github.com/chuanqi305/MobileNet-SSD 下载MobileNet目标检测模型。需要配置文件MobileNetSSD_deploy.prototxt和模型权重MobileNetSSD_deploy.caffemodel。

创建一个空的Android Studio项目并添加OpenCV依赖

参考使用OpenCV进行Android开发教程来初始化您的项目并添加OpenCV。

创建应用

我们的示例将从相机拍摄图片，将其转发到深度神经网络，并接收一系列矩形框、类别标识符和范围在[0, 1]内的置信度值。

首先，我们需要添加一个显示处理帧的必要小部件。修改app/src/main/res/layout/activity_main.xml
<?xml version="1.0" encoding="utf-8"?>

<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"

xmlns:app="http://schemas.android.com/apk/res-auto"

xmlns:tools="http://schemas.android.com/tools"

android:layout_width="match_parent"

android:layout_height="match_parent"

tools:context="org.opencv.samples.opencv_mobilenet.MainActivity">

<org.opencv.android.JavaCameraView

android:id="@+id/CameraView"

android:layout_width="match_parent"

android:layout_height="match_parent"

android:visibility="visible" />

</FrameLayout>
修改/app/src/main/AndroidManifest.xml以启用全屏模式，设置正确的屏幕方向并允许使用相机。
<?xml version="1.0" encoding="utf-8"?>

<manifest xmlns:android="http://schemas.android.com/apk/res/android">

<application

android:label="@string/app_name">

<activity

android:exported="true"

android:name=".MainActivity"

android:screenOrientation="landscape"> 

<intent-filter>

<action android:name="android.intent.action.MAIN" />

<category android:name="android.intent.category.LAUNCHER" />

</intent-filter>

</activity>

</application>



<uses-permission android:name="android.permission.CAMERA"/>

<uses-feature android:name="android.hardware.camera" android:required="false"/>

<uses-feature android:name="android.hardware.camera.autofocus" android:required="false"/>

<uses-feature android:name="android.hardware.camera.front" android:required="false"/>

<uses-feature android:name="android.hardware.camera.front.autofocus" android:required="false"/>

</manifest>
替换app/src/main/java/com/example/myapplication/MainActivity.java的内容，如有必要，请设置自定义包名

package com.example.myapplication;

import android.content.Context;
import android.content.res.AssetManager;
import android.os.Bundle;
import android.util.Log;
import android.widget.Toast;
 
import org.opencv.android.CameraActivity;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Net;
import org.opencv.dnn.Dnn;
import org.opencv.imgproc.Imgproc;
 
import java.io.InputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
 
public class MainActivity extends CameraActivity implements CvCameraViewListener2 {
 
@Override
    public void onResume() {
super.onResume();
        if (mOpenCvCameraView != null)
mOpenCvCameraView.enableView();
    }
 
@Override
    protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
 
        if (OpenCVLoader.initLocal()) {
Log.i(TAG, "OpenCV加载成功");
} else {
Log.e(TAG, "OpenCV初始化失败!");
(Toast.makeText(this, "OpenCV初始化失败!", Toast.LENGTH_LONG)).show();
            return;
        }
 
mModelBuffer = loadFileFromResource(R.raw.mobilenet_iter_73000);
mConfigBuffer = loadFileFromResource(R.raw.deploy);
        if (mModelBuffer == null || mConfigBuffer == null) {
Log.e(TAG, "从资源加载模型失败");
} else
Log.i(TAG, "模型文件加载成功");
 
net = Dnn.readNet("caffe", mModelBuffer, mConfigBuffer);
Log.i(TAG, "网络加载成功");
 
setContentView(R.layout.activity_main);
 
        // 设置相机监听器。
mOpenCvCameraView = (CameraBridgeViewBase)findViewById(R.id.CameraView);
mOpenCvCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
mOpenCvCameraView.setCvCameraViewListener(this);
    }
 
@Override
    public void onPause()
    {
super.onPause();
        if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();
    }
 
@Override
    protected List<? extends CameraBridgeViewBase> getCameraViewList() {
        return Collections.singletonList(mOpenCvCameraView);
    }
 
    public void onDestroy() {
super.onDestroy();
        if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();
 
mModelBuffer.release();
mConfigBuffer.release();
    }
 
    // 加载网络。
    public void onCameraViewStarted(int width, int height) {
    }
 
    public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
        final int IN_WIDTH = 300;
        final int IN_HEIGHT = 300;
        final float WH_RATIO = (float)IN_WIDTH / IN_HEIGHT;
        final double IN_SCALE_FACTOR = 0.007843;
        final double MEAN_VAL = 127.5;
        final double THRESHOLD = 0.2;
 
        // 获取新帧
Log.d(TAG, "处理新帧!");
Mat frame = inputFrame.rgba();
Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB);
 
        // 将图像通过网络前向传播。
Mat blob = Dnn.blobFromImage(frame, IN_SCALE_FACTOR,
                new Size(IN_WIDTH, IN_HEIGHT),
                new Scalar(MEAN_VAL, MEAN_VAL, MEAN_VAL), /*swapRB*/false, /*crop*/false);
net.setInput(blob);
Mat detections = net.forward();
 
        int cols = frame.cols();
        int rows = frame.rows();
 
detections = detections.reshape(1, (int)detections.total() / 7);
 
        for (int i = 0; i < detections.rows(); ++i) {
            double confidence = detections.get(i, 2)[0];
            if (confidence > THRESHOLD) {
                int classId = (int)detections.get(i, 1)[0];
 
                int left = (int)(detections.get(i, 3)[0] * cols);
                int top = (int)(detections.get(i, 4)[0] * rows);
                int right = (int)(detections.get(i, 5)[0] * cols);
                int bottom = (int)(detections.get(i, 6)[0] * rows);
 
                // 在检测到的物体周围绘制矩形。
Imgproc.rectangle(frame, new Point(left, top), new Point(right, bottom),
                                  new Scalar(0, 255, 0));
                String label = classNames[classId] + ": " + confidence;
                int[] baseLine = new int[1];
                Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
 
                // 绘制标签的背景。
Imgproc.rectangle(frame, new Point(left, top - labelSize.height),
                                  new Point(left + labelSize.width, top + baseLine[0]),
                                  new Scalar(255, 255, 255), Imgproc.FILLED);
                // 写入类名和置信度。
Imgproc.putText(frame, label, new Point(left, top),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
            }
        }
 
        return frame;
    }
 
    public void onCameraViewStopped() {}
 
    private MatOfByte loadFileFromResource(int id) {
       byte[] buffer;
        try {
            // 从应用程序资源加载级联文件
InputStream is = getResources().openRawResource(id);
 
            int size = is.available();
buffer = new byte[size];
            int bytesRead = is.read(buffer);
is.close();
} catch (IOException e) {
e.printStackTrace();
Log.e(TAG, "加载ONNX模型失败! 抛出异常: " + e);
(Toast.makeText(this, "加载ONNX模型失败!", Toast.LENGTH_LONG)).show();
            return null;
        }
 
        return new MatOfByte(buffer);
    }
 
    private static final String TAG = "OpenCV-MobileNet";
    private static final String[] classNames = {"背景",
            "飞机", "自行车", "鸟", "船",
            "瓶子", "公交车", "汽车", "猫", "椅子",
            "牛", "餐桌", "狗", "马",
            "摩托车", "人", "盆栽",
            "绵羊", "沙发", "火车", "电视机"};
 
    private MatOfByte mConfigBuffer;
    private MatOfByte mModelBuffer;
    private Net net;
    private CameraBridgeViewBase mOpenCvCameraView;
}

将下载的deploy.prototxt 和 mobilenet_iter_73000.caffemodel 文件放入app/src/main/res/raw文件夹中。OpenCV DNN 模型主要用于从文件中加载 ML 和 DNN 模型。现代 Android 系统不允许在没有额外权限的情况下执行此操作，但提供了 Java API 用于从资源加载字节。此示例使用替代的 DNN API，该 API 从内存缓冲区而不是文件初始化模型。以下函数从资源读取模型文件并将其转换为MatOfBytes（类似于 C++ 中的std::vector<char>）对象，该对象适用于 OpenCV Java API

    private MatOfByte loadFileFromResource(int id) {
       byte[] buffer;
        try {
            // 从应用程序资源加载级联文件
InputStream is = getResources().openRawResource(id);
 
            int size = is.available();
buffer = new byte[size];
            int bytesRead = is.read(buffer);
is.close();
} catch (IOException e) {
e.printStackTrace();
Log.e(TAG, "加载ONNX模型失败! 抛出异常: " + e);
(Toast.makeText(this, "加载ONNX模型失败!", Toast.LENGTH_LONG)).show();
            return null;
        }
 
        return new MatOfByte(buffer);
    }

然后，网络初始化使用以下几行代码完成

mModelBuffer = loadFileFromResource(R.raw.mobilenet_iter_73000);
mConfigBuffer = loadFileFromResource(R.raw.deploy);
        if (mModelBuffer == null || mConfigBuffer == null) {
Log.e(TAG, "从资源加载模型失败");
} else
Log.i(TAG, "模型文件加载成功");
 
net = Dnn.readNet("caffe", mModelBuffer, mConfigBuffer);
Log.i(TAG, "网络加载成功");

另请参见 Android 资源文档

了解如何准备 DNN 模型输入并解释推理结果

Mat blob = Dnn.blobFromImage(frame, IN_SCALE_FACTOR,
                new Size(IN_WIDTH, IN_HEIGHT),
                new Scalar(MEAN_VAL, MEAN_VAL, MEAN_VAL), /*swapRB*/false, /*crop*/false);
net.setInput(blob);
Mat detections = net.forward();
 
        int cols = frame.cols();
        int rows = frame.rows();
 
detections = detections.reshape(1, (int)detections.total() / 7);
 
        for (int i = 0; i < detections.rows(); ++i) {
            double confidence = detections.get(i, 2)[0];
            if (confidence > THRESHOLD) {
                int classId = (int)detections.get(i, 1)[0];
 
                int left = (int)(detections.get(i, 3)[0] * cols);
                int top = (int)(detections.get(i, 4)[0] * rows);
                int right = (int)(detections.get(i, 5)[0] * cols);
                int bottom = (int)(detections.get(i, 6)[0] * rows);
 
                // 在检测到的物体周围绘制矩形。
Imgproc.rectangle(frame, new Point(left, top), new Point(right, bottom),
                                  new Scalar(0, 255, 0));
                String label = classNames[classId] + ": " + confidence;
                int[] baseLine = new int[1];
                Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
 
                // 绘制标签的背景。
Imgproc.rectangle(frame, new Point(left, top - labelSize.height),
                                  new Point(left + labelSize.width, top + baseLine[0]),
                                  new Scalar(255, 255, 255), Imgproc.FILLED);
                // 写入类名和置信度。
Imgproc.putText(frame, label, new Point(left, top),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
            }
        }

Dnn.blobFromImage 将摄像机帧转换为神经网络输入张量。应用了调整大小和统计归一化。网络输出张量的每一行包含关于检测到的一个对象的信息，顺序如下：置信度（范围为[0, 1]）、类别 ID、左上角坐标、右下角坐标。所有坐标都在[0, 1]范围内，应在渲染前缩放为图像大小。

启动应用程序，享受乐趣！


原作者	Dmitry Kurtaev
兼容性	OpenCV >= 4.9

目录

简介

需求

创建一个空的Android Studio项目并添加OpenCV依赖

创建应用