如何在 Android 设备上运行深度网络

目录

• 简介
• 要求
• 创建一个空的Android Studio项目并添加OpenCV依赖
• 制作一个应用

上一个教程： 使用OpenCV进行Android开发
下一个教程： 在基于Android相机预览的CV应用中使用OpenCL

另请参见

深度神经网络 (dnn 模块)

原始作者	Dmitry Kurtaev
兼容性	OpenCV >= 4.9

简介

在本教程中，你将了解如何使用OpenCV深度学习模块在Android设备上运行深度学习网络。本教程是为Android Studio 2022.2.1编写的。

要求

• 从https://developer.android.com.cn/studio下载并安装Android Studio。
• 从https://github.com/opencv/opencv/releases获取最新预构建的Android版OpenCV版本并解压（例如，opencv-4.X.Y-android-sdk.zip，需要最低版本4.9）。
• 从https://github.com/chuanqi305/MobileNet-SSD下载MobileNet对象检测模型。需要配置文件MobileNetSSD_deploy.prototxt和模型权重MobileNetSSD_deploy.caffemodel。

创建一个空的Android Studio项目并添加OpenCV依赖

使用使用OpenCV进行Android开发教程来初始化你的项目并添加OpenCV。

制作一个应用

我们的示例将从摄像头获取图像，将其转发到深度网络，并接收一组矩形、类别标识符和范围为[0, 1]的置信度值。

• 首先，我们需要添加一个必要的部件来显示处理后的帧。修改app/src/main/res/layout/activity_main.xml

  <?xml version="1.0" encoding="utf-8"?>
  <FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
      xmlns:app="http://schemas.android.com/apk/res-auto"
      xmlns:tools="http://schemas.android.com/tools"
      android:layout_width="match_parent"
      android:layout_height="match_parent"
      tools:context="org.opencv.samples.opencv_mobilenet.MainActivity">
   
  <org.opencv.android.JavaCameraView
          android:id="@+id/CameraView"
          android:layout_width="match_parent"
          android:layout_height="match_parent"
          android:visibility="visible" />
  </FrameLayout>

• 修改/app/src/main/AndroidManifest.xml以启用全屏模式，设置正确的屏幕方向并允许使用摄像头。

  <?xml version="1.0" encoding="utf-8"?>
  <manifest xmlns:android="http://schemas.android.com/apk/res/android">
   
  <application
          android:label="@string/app_name">

  <activity
                    android:exported="true"
                    android:name=".MainActivity"
                    android:screenOrientation="landscape"> <!--屏幕方向-->
  <intent-filter>
  <action android:name="android.intent.action.MAIN" />
  <category android:name="android.intent.category.LAUNCHER" />
  </intent-filter>
  </activity>
  </application>
   
      <!--允许使用摄像头-->
  <uses-permission android:name="android.permission.CAMERA"/>
  <uses-feature android:name="android.hardware.camera" android:required="false"/>
  <uses-feature android:name="android.hardware.camera.autofocus" android:required="false"/>
  <uses-feature android:name="android.hardware.camera.front" android:required="false"/>
  <uses-feature android:name="android.hardware.camera.front.autofocus" android:required="false"/>
   
  </manifest>

• 替换app/src/main/java/com/example/myapplication/MainActivity.java的内容，并在必要时设置自定义包名

package com.example.myapplication;

import android.content.Context;
import android.content.res.AssetManager;
import android.os.Bundle;
import android.util.Log;
import android.widget.Toast;
 
import org.opencv.android.CameraActivity;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Net;
import org.opencv.dnn.Dnn;
import org.opencv.imgproc.Imgproc;
 
import java.io.InputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
 
public class MainActivity extends CameraActivity implements CvCameraViewListener2 {
 
@Override
    public void onResume() {
super.onResume();
        if (mOpenCvCameraView != null)
mOpenCvCameraView.enableView();
    }
 
@Override
    protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
 
        if (OpenCVLoader.initLocal()) {
Log.i(TAG, "OpenCV loaded successfully");
} else {
Log.e(TAG, "OpenCV initialization failed!");
(Toast.makeText(this, "OpenCV initialization failed!", Toast.LENGTH_LONG)).show();
            return;
        }
 
mModelBuffer = loadFileFromResource(R.raw.mobilenet_iter_73000);
mConfigBuffer = loadFileFromResource(R.raw.deploy);
        if (mModelBuffer == null || mConfigBuffer == null) {
Log.e(TAG, "Failed to load model from resources");
} else
Log.i(TAG, "Model files loaded successfully");
 
net = Dnn.readNet("caffe", mModelBuffer, mConfigBuffer);
Log.i(TAG, "Network loaded successfully");
 
setContentView(R.layout.activity_main);
 
        // 设置摄像头监听器。
mOpenCvCameraView = (CameraBridgeViewBase)findViewById(R.id.CameraView);
mOpenCvCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
mOpenCvCameraView.setCvCameraViewListener(this);
    }
 
@Override
    public void onPause()
    {
super.onPause();
        if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();
    }
 
@Override
    protected List<? extends CameraBridgeViewBase> getCameraViewList() {
        return Collections.singletonList(mOpenCvCameraView);
    }
 
    public void onDestroy() {
super.onDestroy();
        if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();
 
mModelBuffer.release();
mConfigBuffer.release();
    }
 
    // 加载网络。
    public void onCameraViewStarted(int width, int height) {
    }
 
    public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
        final int IN_WIDTH = 300;
        final int IN_HEIGHT = 300;
        final float WH_RATIO = (float)IN_WIDTH / IN_HEIGHT;
        final double IN_SCALE_FACTOR = 0.007843;
        final double MEAN_VAL = 127.5;
        final double THRESHOLD = 0.2;
 
        // 获取新帧
Log.d(TAG, "handle new frame!");
Mat frame = inputFrame.rgba();
Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB);
 
        // 将图像转发到网络。
Mat blob = Dnn.blobFromImage(frame, IN_SCALE_FACTOR,
                new Size(IN_WIDTH, IN_HEIGHT),
                new Scalar(MEAN_VAL, MEAN_VAL, MEAN_VAL), /*swapRB*/false, /*crop*/false);
net.setInput(blob);
Mat detections = net.forward();
 
        int cols = frame.cols();
        int rows = frame.rows();
 
detections = detections.reshape(1, (int)detections.total() / 7);
 
        for (int i = 0; i < detections.rows(); ++i) {
            double confidence = detections.get(i, 2)[0];
            if (confidence > THRESHOLD) {
                int classId = (int)detections.get(i, 1)[0];
 
                int left = (int)(detections.get(i, 3)[0] * cols);
                int top = (int)(detections.get(i, 4)[0] * rows);
                int right = (int)(detections.get(i, 5)[0] * cols);
                int bottom = (int)(detections.get(i, 6)[0] * rows);
 
                // 在检测到的对象周围绘制矩形。
Imgproc.rectangle(frame, new Point(left, top), new Point(right, bottom),
                                  new Scalar(0, 255, 0));
                String label = classNames[classId] + ": " + confidence;
                int[] baseLine = new int[1];
                Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
 
                // 绘制标签背景。
Imgproc.rectangle(frame, new Point(left, top - labelSize.height),
                                  new Point(left + labelSize.width, top + baseLine[0]),
                                  new Scalar(255, 255, 255), Imgproc.FILLED);
                // 写入类别名称和置信度。
Imgproc.putText(frame, label, new Point(left, top),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
            }
        }
 
        return frame;
    }
 
    public void onCameraViewStopped() {}
 
    private MatOfByte loadFileFromResource(int id) {
       byte[] buffer;
        try {
            // 从应用程序资源加载级联文件
InputStream is = getResources().openRawResource(id);
 
            int size = is.available();
buffer = new byte[size];
            int bytesRead = is.read(buffer);
is.close();
} catch (IOException e) {
e.printStackTrace();
Log.e(TAG, "Failed to ONNX model from resources! Exception thrown: " + e);
(Toast.makeText(this, "Failed to ONNX model from resources!", Toast.LENGTH_LONG)).show();
            return null;
        }
 
        return new MatOfByte(buffer);
    }
 
    private static final String TAG = "OpenCV-MobileNet";
    private static final String[] classNames = {"background",
            "aeroplane", "bicycle", "bird", "boat",
            "bottle", "bus", "car", "cat", "chair",
            "cow", "diningtable", "dog", "horse",
            "motorbike", "person", "pottedplant",
            "sheep", "sofa", "train", "tvmonitor"};
 
    private MatOfByte mConfigBuffer;
    private MatOfByte mModelBuffer;
    private Net net;
    private CameraBridgeViewBase mOpenCvCameraView;
}

• 将下载的deploy.prototxt和mobilenet_iter_73000.caffemodel放入app/src/main/res/raw文件夹。OpenCV DNN模型主要设计用于从文件加载ML和DNN模型。现代Android在没有额外权限的情况下不允许这样做，但提供了Java API来从资源加载字节。该示例使用了替代的DNN API，该API从内存缓冲区而不是文件初始化模型。以下函数从资源中读取模型文件并将其转换为适用于OpenCV Java API的MatOfBytes对象（C++世界中std::vector<char>的对应物）

    private MatOfByte loadFileFromResource(int id) {
       byte[] buffer;
        try {
            // 从应用程序资源加载级联文件
InputStream is = getResources().openRawResource(id);
 
            int size = is.available();
buffer = new byte[size];
            int bytesRead = is.read(buffer);
is.close();
} catch (IOException e) {
e.printStackTrace();
Log.e(TAG, "Failed to ONNX model from resources! Exception thrown: " + e);
(Toast.makeText(this, "Failed to ONNX model from resources!", Toast.LENGTH_LONG)).show();
            return null;
        }
 
        return new MatOfByte(buffer);
    }

然后通过以下几行代码完成网络初始化

mModelBuffer = loadFileFromResource(R.raw.mobilenet_iter_73000);
mConfigBuffer = loadFileFromResource(R.raw.deploy);
        if (mModelBuffer == null || mConfigBuffer == null) {
Log.e(TAG, "Failed to load model from resources");
} else
Log.i(TAG, "Model files loaded successfully");
 
net = Dnn.readNet("caffe", mModelBuffer, mConfigBuffer);
Log.i(TAG, "Network loaded successfully");

另请参阅Android资源文档

• 看看DNN模型输入是如何准备的以及推理结果是如何解释的

Mat blob = Dnn.blobFromImage(frame, IN_SCALE_FACTOR,
                new Size(IN_WIDTH, IN_HEIGHT),
                new Scalar(MEAN_VAL, MEAN_VAL, MEAN_VAL), /*swapRB*/false, /*crop*/false);
net.setInput(blob);
Mat detections = net.forward();
 
        int cols = frame.cols();
        int rows = frame.rows();
 
detections = detections.reshape(1, (int)detections.total() / 7);
 
        for (int i = 0; i < detections.rows(); ++i) {
            double confidence = detections.get(i, 2)[0];
            if (confidence > THRESHOLD) {
                int classId = (int)detections.get(i, 1)[0];
 
                int left = (int)(detections.get(i, 3)[0] * cols);
                int top = (int)(detections.get(i, 4)[0] * rows);
                int right = (int)(detections.get(i, 5)[0] * cols);
                int bottom = (int)(detections.get(i, 6)[0] * rows);
 
                // 在检测到的对象周围绘制矩形。
Imgproc.rectangle(frame, new Point(left, top), new Point(right, bottom),
                                  new Scalar(0, 255, 0));
                String label = classNames[classId] + ": " + confidence;
                int[] baseLine = new int[1];
                Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
 
                // 绘制标签背景。
Imgproc.rectangle(frame, new Point(left, top - labelSize.height),
                                  new Point(left + labelSize.width, top + baseLine[0]),
                                  new Scalar(255, 255, 255), Imgproc.FILLED);
                // 写入类别名称和置信度。
Imgproc.putText(frame, label, new Point(left, top),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
            }
        }

Dnn.blobFromImage将相机帧转换为神经网络输入张量。应用了大小调整和统计归一化。网络输出张量的每一行都包含一个检测到的对象的信息，顺序如下：置信度范围[0, 1]、类别ID、左、上、右、下框坐标。所有坐标都在[0, 1]范围内，并在渲染前应按图像大小进行缩放。

• 启动应用程序，尽情玩乐吧！