266 lines
10 KiB
C#
Executable File
266 lines
10 KiB
C#
Executable File
#if !UNITY_WSA_10_0
|
|
|
|
using UnityEngine;
|
|
using UnityEngine.SceneManagement;
|
|
using System;
|
|
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using System.Xml;
|
|
using OpenCVForUnity.CoreModule;
|
|
using OpenCVForUnity.ImgcodecsModule;
|
|
using OpenCVForUnity.TextModule;
|
|
using OpenCVForUnity.ImgprocModule;
|
|
using OpenCVForUnity.UnityUtils;
|
|
|
|
namespace OpenCVForUnityExample
|
|
{
|
|
/// <summary>
|
|
/// Text Detection Example
|
|
/// A demo script of the Extremal Region Filter algorithm described in:Neumann L., Matas J.: Real-Time Scene Text Localization and Recognition, CVPR 2012.
|
|
/// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/text/samples/textdetection.py.
|
|
/// </summary>
|
|
public class TextRecognitionExample : MonoBehaviour
|
|
{
|
|
string scenetext01_jpg_filepath;
|
|
string trained_classifierNM1_xml_filepath;
|
|
string trained_classifierNM2_xml_filepath;
|
|
string OCRHMM_transitions_table_xml_filepath;
|
|
string OCRHMM_knn_model_data_xml_gz_filepath;
|
|
|
|
#if UNITY_WEBGL && !UNITY_EDITOR
|
|
IEnumerator getFilePath_Coroutine;
|
|
#endif
|
|
|
|
|
|
// Use this for initialization
|
|
void Start ()
|
|
{
|
|
#if UNITY_WEBGL && !UNITY_EDITOR
|
|
getFilePath_Coroutine = GetFilePath ();
|
|
StartCoroutine (getFilePath_Coroutine);
|
|
#else
|
|
scenetext01_jpg_filepath = Utils.getFilePath ("text/test_text.jpg");
|
|
trained_classifierNM1_xml_filepath = Utils.getFilePath ("text/trained_classifierNM1.xml");
|
|
trained_classifierNM2_xml_filepath = Utils.getFilePath ("text/trained_classifierNM2.xml");
|
|
OCRHMM_transitions_table_xml_filepath = Utils.getFilePath ("text/OCRHMM_transitions_table.xml");
|
|
#if UNITY_ANDROID && !UNITY_EDITOR
|
|
OCRHMM_knn_model_data_xml_gz_filepath = Utils.getFilePath ("text/OCRHMM_knn_model_data.xml");
|
|
#else
|
|
OCRHMM_knn_model_data_xml_gz_filepath = Utils.getFilePath ("text/OCRHMM_knn_model_data.xml.gz");
|
|
#endif
|
|
Run ();
|
|
#endif
|
|
}
|
|
|
|
#if UNITY_WEBGL && !UNITY_EDITOR
|
|
private IEnumerator GetFilePath ()
|
|
{
|
|
var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync ("text/test_text.jpg", (result) => {
|
|
scenetext01_jpg_filepath = result;
|
|
});
|
|
yield return getFilePathAsync_0_Coroutine;
|
|
|
|
var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync ("text/trained_classifierNM1.xml", (result) => {
|
|
trained_classifierNM1_xml_filepath = result;
|
|
});
|
|
yield return getFilePathAsync_1_Coroutine;
|
|
|
|
var getFilePathAsync_2_Coroutine = Utils.getFilePathAsync ("text/trained_classifierNM2.xml", (result) => {
|
|
trained_classifierNM2_xml_filepath = result;
|
|
});
|
|
yield return getFilePathAsync_2_Coroutine;
|
|
|
|
var getFilePathAsync_3_Coroutine = Utils.getFilePathAsync ("text/OCRHMM_transitions_table.xml", (result) => {
|
|
OCRHMM_transitions_table_xml_filepath = result;
|
|
});
|
|
yield return getFilePathAsync_3_Coroutine;
|
|
|
|
var getFilePathAsync_4_Coroutine = Utils.getFilePathAsync ("text/OCRHMM_knn_model_data.xml.gz", (result) => {
|
|
OCRHMM_knn_model_data_xml_gz_filepath = result;
|
|
});
|
|
yield return getFilePathAsync_4_Coroutine;
|
|
|
|
getFilePath_Coroutine = null;
|
|
|
|
Run ();
|
|
}
|
|
#endif
|
|
|
|
private void Run ()
|
|
{
|
|
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
|
|
Utils.setDebugMode (true);
|
|
|
|
|
|
Mat frame = Imgcodecs.imread (scenetext01_jpg_filepath);
|
|
#if !UNITY_WSA_10_0
|
|
if (frame.empty ()) {
|
|
Debug.LogError ("text/scenetext01.jpg is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
|
|
}
|
|
#endif
|
|
|
|
Mat binaryMat = new Mat ();
|
|
Mat maskMat = new Mat ();
|
|
|
|
|
|
List<MatOfPoint> regions = new List<MatOfPoint> ();
|
|
|
|
ERFilter er_filter1 = Text.createERFilterNM1 (trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
|
|
|
|
ERFilter er_filter2 = Text.createERFilterNM2 (trained_classifierNM2_xml_filepath, 0.5f);
|
|
|
|
|
|
Mat transition_p = new Mat (62, 62, CvType.CV_64FC1);
|
|
// string filename = "OCRHMM_transitions_table.xml";
|
|
// FileStorage fs(filename, FileStorage::READ);
|
|
// fs["transition_probabilities"] >> transition_p;
|
|
// fs.release();
|
|
|
|
//Load TransitionProbabilitiesData.
|
|
transition_p.put (0, 0, GetTransitionProbabilitiesData (OCRHMM_transitions_table_xml_filepath));
|
|
|
|
Mat emission_p = Mat.eye (62, 62, CvType.CV_64FC1);
|
|
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
|
|
OCRHMMDecoder decoder = OCRHMMDecoder.create (
|
|
OCRHMM_knn_model_data_xml_gz_filepath,
|
|
voc, transition_p, emission_p);
|
|
|
|
//Text Detection
|
|
Imgproc.cvtColor (frame, frame, Imgproc.COLOR_BGR2RGB);
|
|
Imgproc.cvtColor (frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
|
|
Imgproc.threshold (binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
|
|
Core.absdiff (binaryMat, new Scalar (255), maskMat);
|
|
|
|
|
|
Text.detectRegions (binaryMat, er_filter1, er_filter2, regions);
|
|
Debug.Log ("regions.Count " + regions.Count);
|
|
|
|
|
|
MatOfRect groups_rects = new MatOfRect ();
|
|
List<OpenCVForUnity.CoreModule.Rect> rects = new List<OpenCVForUnity.CoreModule.Rect> ();
|
|
Text.erGrouping (frame, binaryMat, regions, groups_rects);
|
|
|
|
|
|
for (int i = 0; i < regions.Count; i++) {
|
|
regions [i].Dispose ();
|
|
}
|
|
regions.Clear ();
|
|
|
|
|
|
rects.AddRange (groups_rects.toList ());
|
|
|
|
groups_rects.Dispose ();
|
|
|
|
|
|
//Text Recognition (OCR)
|
|
|
|
List<Mat> detections = new List<Mat> ();
|
|
|
|
for (int i = 0; i < (int)rects.Count; i++) {
|
|
|
|
Mat group_img = new Mat ();
|
|
maskMat.submat (rects [i]).copyTo (group_img);
|
|
Core.copyMakeBorder (group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar (0));
|
|
detections.Add (group_img);
|
|
}
|
|
|
|
Debug.Log ("detections.Count " + detections.Count);
|
|
|
|
|
|
//#Visualization
|
|
for (int i = 0; i < rects.Count; i++) {
|
|
|
|
|
|
Imgproc.rectangle (frame, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (255, 0, 0), 2);
|
|
Imgproc.rectangle (frame, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (255, 255, 255), 1);
|
|
|
|
string output = decoder.run (detections [i], 0);
|
|
if (!string.IsNullOrEmpty (output)) {
|
|
Debug.Log ("output " + output);
|
|
Imgproc.putText (frame, output, new Point (rects [i].x, rects [i].y), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (0, 0, 255), 1, Imgproc.LINE_AA, false);
|
|
}
|
|
}
|
|
|
|
|
|
Texture2D texture = new Texture2D (frame.cols (), frame.rows (), TextureFormat.RGBA32, false);
|
|
|
|
Utils.matToTexture2D (frame, texture);
|
|
|
|
// Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
|
|
//
|
|
// Utils.matToTexture2D (detections [0], texture);
|
|
|
|
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
|
|
|
|
|
|
for (int i = 0; i < detections.Count; i++) {
|
|
detections [i].Dispose ();
|
|
}
|
|
binaryMat.Dispose ();
|
|
maskMat.Dispose ();
|
|
|
|
Utils.setDebugMode (false);
|
|
}
|
|
|
|
// Update is called once per frame
|
|
void Update ()
|
|
{
|
|
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the transition probabilities data.
|
|
/// </summary>
|
|
/// <returns>The transition probabilities data.</returns>
|
|
/// <param name="filePath">File path.</param>
|
|
double[] GetTransitionProbabilitiesData (string filePath)
|
|
{
|
|
XmlDocument xmlDoc = new XmlDocument ();
|
|
xmlDoc.Load (filePath);
|
|
|
|
|
|
XmlNode dataNode = xmlDoc.GetElementsByTagName ("data").Item (0);
|
|
// Debug.Log ("dataNode.InnerText " + dataNode.InnerText);
|
|
string[] dataString = dataNode.InnerText.Split (new string[] {
|
|
" ",
|
|
"\r\n", "\n"
|
|
}, StringSplitOptions.RemoveEmptyEntries);
|
|
// Debug.Log ("dataString.Length " + dataString.Length);
|
|
|
|
double[] data = new double[dataString.Length];
|
|
for (int i = 0; i < data.Length; i++) {
|
|
try {
|
|
data [i] = Convert.ToDouble (dataString [i]);
|
|
} catch (FormatException) {
|
|
Debug.Log ("Unable to convert '{" + dataString [i] + "}' to a Double.");
|
|
} catch (OverflowException) {
|
|
Debug.Log ("'{" + dataString [i] + "}' is outside the range of a Double.");
|
|
}
|
|
}
|
|
|
|
return data;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Raises the destroy event.
|
|
/// </summary>
|
|
void OnDestroy ()
|
|
{
|
|
#if UNITY_WEBGL && !UNITY_EDITOR
|
|
if (getFilePath_Coroutine != null) {
|
|
StopCoroutine (getFilePath_Coroutine);
|
|
((IDisposable)getFilePath_Coroutine).Dispose ();
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/// <summary>
|
|
/// Raises the back button click event.
|
|
/// </summary>
|
|
public void OnBackButtonClick ()
|
|
{
|
|
SceneManager.LoadScene ("OpenCVForUnityExample");
|
|
}
|
|
}
|
|
}
|
|
#endif |