#if !UNITY_WSA_10_0
using UnityEngine;
using UnityEngine.SceneManagement;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Xml;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgcodecsModule;
using OpenCVForUnity.TextModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.UnityUtils;
namespace OpenCVForUnityExample
{
/// <summary>
/// Text Detection Example
/// A demo script of the Extremal Region Filter algorithm described in:Neumann L., Matas J.: Real-Time Scene Text Localization and Recognition, CVPR 2012.
/// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/text/samples/textdetection.py.
/// </summary>
public class TextRecognitionExample : MonoBehaviour
{
string scenetext01_jpg_filepath;
string trained_classifierNM1_xml_filepath;
string trained_classifierNM2_xml_filepath;
string OCRHMM_transitions_table_xml_filepath;
string OCRHMM_knn_model_data_xml_gz_filepath;
#if UNITY_WEBGL && !UNITY_EDITOR
IEnumerator getFilePath_Coroutine;
#endif
// Use this for initialization
void Start ()
{
#if UNITY_WEBGL && !UNITY_EDITOR
getFilePath_Coroutine = GetFilePath ();
StartCoroutine (getFilePath_Coroutine);
#else
scenetext01_jpg_filepath = Utils.getFilePath ("text/test_text.jpg");
trained_classifierNM1_xml_filepath = Utils.getFilePath ("text/trained_classifierNM1.xml");
trained_classifierNM2_xml_filepath = Utils.getFilePath ("text/trained_classifierNM2.xml");
OCRHMM_transitions_table_xml_filepath = Utils.getFilePath ("text/OCRHMM_transitions_table.xml");
#if UNITY_ANDROID && !UNITY_EDITOR
OCRHMM_knn_model_data_xml_gz_filepath = Utils.getFilePath ("text/OCRHMM_knn_model_data.xml");
#else
OCRHMM_knn_model_data_xml_gz_filepath = Utils.getFilePath ("text/OCRHMM_knn_model_data.xml.gz");
#endif
Run ();
#endif
}
#if UNITY_WEBGL && !UNITY_EDITOR
private IEnumerator GetFilePath ()
{
var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync ("text/test_text.jpg", (result) => {
scenetext01_jpg_filepath = result;
});
yield return getFilePathAsync_0_Coroutine;
var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync ("text/trained_classifierNM1.xml", (result) => {
trained_classifierNM1_xml_filepath = result;
});
yield return getFilePathAsync_1_Coroutine;
var getFilePathAsync_2_Coroutine = Utils.getFilePathAsync ("text/trained_classifierNM2.xml", (result) => {
trained_classifierNM2_xml_filepath = result;
});
yield return getFilePathAsync_2_Coroutine;
var getFilePathAsync_3_Coroutine = Utils.getFilePathAsync ("text/OCRHMM_transitions_table.xml", (result) => {
OCRHMM_transitions_table_xml_filepath = result;
});
yield return getFilePathAsync_3_Coroutine;
var getFilePathAsync_4_Coroutine = Utils.getFilePathAsync ("text/OCRHMM_knn_model_data.xml.gz", (result) => {
OCRHMM_knn_model_data_xml_gz_filepath = result;
});
yield return getFilePathAsync_4_Coroutine;
getFilePath_Coroutine = null;
Run ();
}
#endif
private void Run ()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode (true);
Mat frame = Imgcodecs.imread (scenetext01_jpg_filepath);
#if !UNITY_WSA_10_0
if (frame.empty ()) {
Debug.LogError ("text/scenetext01.jpg is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. ");
}
#endif
Mat binaryMat = new Mat ();
Mat maskMat = new Mat ();
List<MatOfPoint> regions = new List<MatOfPoint> ();
ERFilter er_filter1 = Text.createERFilterNM1 (trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f);
ERFilter er_filter2 = Text.createERFilterNM2 (trained_classifierNM2_xml_filepath, 0.5f);
Mat transition_p = new Mat (62, 62, CvType.CV_64FC1);
// string filename = "OCRHMM_transitions_table.xml";
// FileStorage fs(filename, FileStorage::READ);
// fs["transition_probabilities"] >> transition_p;
// fs.release();
//Load TransitionProbabilitiesData.
transition_p.put (0, 0, GetTransitionProbabilitiesData (OCRHMM_transitions_table_xml_filepath));
Mat emission_p = Mat.eye (62, 62, CvType.CV_64FC1);
string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
OCRHMMDecoder decoder = OCRHMMDecoder.create (
OCRHMM_knn_model_data_xml_gz_filepath,
voc, transition_p, emission_p);
//Text Detection
Imgproc.cvtColor (frame, frame, Imgproc.COLOR_BGR2RGB);
Imgproc.cvtColor (frame, binaryMat, Imgproc.COLOR_RGB2GRAY);
Imgproc.threshold (binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
Core.absdiff (binaryMat, new Scalar (255), maskMat);
Text.detectRegions (binaryMat, er_filter1, er_filter2, regions);
Debug.Log ("regions.Count " + regions.Count);
MatOfRect groups_rects = new MatOfRect ();
List<OpenCVForUnity.CoreModule.Rect> rects = new List<OpenCVForUnity.CoreModule.Rect> ();
Text.erGrouping (frame, binaryMat, regions, groups_rects);
for (int i = 0; i < regions.Count; i++) {
regions [i].Dispose ();
}
regions.Clear ();
rects.AddRange (groups_rects.toList ());
groups_rects.Dispose ();
//Text Recognition (OCR)
List<Mat> detections = new List<Mat> ();
for (int i = 0; i < (int)rects.Count; i++) {
Mat group_img = new Mat ();
maskMat.submat (rects [i]).copyTo (group_img);
Core.copyMakeBorder (group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar (0));
detections.Add (group_img);
}
Debug.Log ("detections.Count " + detections.Count);
//#Visualization
for (int i = 0; i < rects.Count; i++) {
Imgproc.rectangle (frame, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (255, 0, 0), 2);
Imgproc.rectangle (frame, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (255, 255, 255), 1);
string output = decoder.run (detections [i], 0);
if (!string.IsNullOrEmpty (output)) {
Debug.Log ("output " + output);
Imgproc.putText (frame, output, new Point (rects [i].x, rects [i].y), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (0, 0, 255), 1, Imgproc.LINE_AA, false);
}
}
Texture2D texture = new Texture2D (frame.cols (), frame.rows (), TextureFormat.RGBA32, false);
Utils.matToTexture2D (frame, texture);
// Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false);
//
// Utils.matToTexture2D (detections [0], texture);
gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
for (int i = 0; i < detections.Count; i++) {
detections [i].Dispose ();
}
binaryMat.Dispose ();
maskMat.Dispose ();
Utils.setDebugMode (false);
}
// Update is called once per frame
void Update ()
{
}
/// <summary>
/// Gets the transition probabilities data.
/// </summary>
/// <returns>The transition probabilities data.</returns>
/// <param name="filePath">File path.</param>
double[] GetTransitionProbabilitiesData (string filePath)
{
XmlDocument xmlDoc = new XmlDocument ();
xmlDoc.Load (filePath);
XmlNode dataNode = xmlDoc.GetElementsByTagName ("data").Item (0);
// Debug.Log ("dataNode.InnerText " + dataNode.InnerText);
string[] dataString = dataNode.InnerText.Split (new string[] {
" ",
"\r\n", "\n"
}, StringSplitOptions.RemoveEmptyEntries);
// Debug.Log ("dataString.Length " + dataString.Length);
double[] data = new double[dataString.Length];
for (int i = 0; i < data.Length; i++) {
try {
data [i] = Convert.ToDouble (dataString [i]);
} catch (FormatException) {
Debug.Log ("Unable to convert '{" + dataString [i] + "}' to a Double.");
} catch (OverflowException) {
Debug.Log ("'{" + dataString [i] + "}' is outside the range of a Double.");
}
}
return data;
}
/// <summary>
/// Raises the destroy event.
/// </summary>
void OnDestroy ()
{
#if UNITY_WEBGL && !UNITY_EDITOR
if (getFilePath_Coroutine != null) {
StopCoroutine (getFilePath_Coroutine);
((IDisposable)getFilePath_Coroutine).Dispose ();
}
#endif
}
/// <summary>
/// Raises the back button click event.
/// </summary>
public void OnBackButtonClick ()
{
SceneManager.LoadScene ("OpenCVForUnityExample");
}
}
}
#endif