#if !UNITY_WSA_10_0 using UnityEngine; using UnityEngine.SceneManagement; using System; using System.Collections; using System.Collections.Generic; using System.Xml; using OpenCVForUnity.CoreModule; using OpenCVForUnity.ImgcodecsModule; using OpenCVForUnity.TextModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.UnityUtils; namespace OpenCVForUnityExample { /// /// Text Detection Example /// A demo script of the Extremal Region Filter algorithm described in:Neumann L., Matas J.: Real-Time Scene Text Localization and Recognition, CVPR 2012. /// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/text/samples/textdetection.py. /// public class TextRecognitionExample : MonoBehaviour { string scenetext01_jpg_filepath; string trained_classifierNM1_xml_filepath; string trained_classifierNM2_xml_filepath; string OCRHMM_transitions_table_xml_filepath; string OCRHMM_knn_model_data_xml_gz_filepath; #if UNITY_WEBGL && !UNITY_EDITOR IEnumerator getFilePath_Coroutine; #endif // Use this for initialization void Start () { #if UNITY_WEBGL && !UNITY_EDITOR getFilePath_Coroutine = GetFilePath (); StartCoroutine (getFilePath_Coroutine); #else scenetext01_jpg_filepath = Utils.getFilePath ("text/test_text.jpg"); trained_classifierNM1_xml_filepath = Utils.getFilePath ("text/trained_classifierNM1.xml"); trained_classifierNM2_xml_filepath = Utils.getFilePath ("text/trained_classifierNM2.xml"); OCRHMM_transitions_table_xml_filepath = Utils.getFilePath ("text/OCRHMM_transitions_table.xml"); #if UNITY_ANDROID && !UNITY_EDITOR OCRHMM_knn_model_data_xml_gz_filepath = Utils.getFilePath ("text/OCRHMM_knn_model_data.xml"); #else OCRHMM_knn_model_data_xml_gz_filepath = Utils.getFilePath ("text/OCRHMM_knn_model_data.xml.gz"); #endif Run (); #endif } #if UNITY_WEBGL && !UNITY_EDITOR private IEnumerator GetFilePath () { var getFilePathAsync_0_Coroutine = Utils.getFilePathAsync ("text/test_text.jpg", (result) => { scenetext01_jpg_filepath = result; }); yield return getFilePathAsync_0_Coroutine; var getFilePathAsync_1_Coroutine = Utils.getFilePathAsync ("text/trained_classifierNM1.xml", (result) => { trained_classifierNM1_xml_filepath = result; }); yield return getFilePathAsync_1_Coroutine; var getFilePathAsync_2_Coroutine = Utils.getFilePathAsync ("text/trained_classifierNM2.xml", (result) => { trained_classifierNM2_xml_filepath = result; }); yield return getFilePathAsync_2_Coroutine; var getFilePathAsync_3_Coroutine = Utils.getFilePathAsync ("text/OCRHMM_transitions_table.xml", (result) => { OCRHMM_transitions_table_xml_filepath = result; }); yield return getFilePathAsync_3_Coroutine; var getFilePathAsync_4_Coroutine = Utils.getFilePathAsync ("text/OCRHMM_knn_model_data.xml.gz", (result) => { OCRHMM_knn_model_data_xml_gz_filepath = result; }); yield return getFilePathAsync_4_Coroutine; getFilePath_Coroutine = null; Run (); } #endif private void Run () { //if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console. Utils.setDebugMode (true); Mat frame = Imgcodecs.imread (scenetext01_jpg_filepath); #if !UNITY_WSA_10_0 if (frame.empty ()) { Debug.LogError ("text/scenetext01.jpg is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/text/” to “Assets/StreamingAssets/” folder. "); } #endif Mat binaryMat = new Mat (); Mat maskMat = new Mat (); List regions = new List (); ERFilter er_filter1 = Text.createERFilterNM1 (trained_classifierNM1_xml_filepath, 8, 0.00015f, 0.13f, 0.2f, true, 0.1f); ERFilter er_filter2 = Text.createERFilterNM2 (trained_classifierNM2_xml_filepath, 0.5f); Mat transition_p = new Mat (62, 62, CvType.CV_64FC1); // string filename = "OCRHMM_transitions_table.xml"; // FileStorage fs(filename, FileStorage::READ); // fs["transition_probabilities"] >> transition_p; // fs.release(); //Load TransitionProbabilitiesData. transition_p.put (0, 0, GetTransitionProbabilitiesData (OCRHMM_transitions_table_xml_filepath)); Mat emission_p = Mat.eye (62, 62, CvType.CV_64FC1); string voc = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; OCRHMMDecoder decoder = OCRHMMDecoder.create ( OCRHMM_knn_model_data_xml_gz_filepath, voc, transition_p, emission_p); //Text Detection Imgproc.cvtColor (frame, frame, Imgproc.COLOR_BGR2RGB); Imgproc.cvtColor (frame, binaryMat, Imgproc.COLOR_RGB2GRAY); Imgproc.threshold (binaryMat, binaryMat, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU); Core.absdiff (binaryMat, new Scalar (255), maskMat); Text.detectRegions (binaryMat, er_filter1, er_filter2, regions); Debug.Log ("regions.Count " + regions.Count); MatOfRect groups_rects = new MatOfRect (); List rects = new List (); Text.erGrouping (frame, binaryMat, regions, groups_rects); for (int i = 0; i < regions.Count; i++) { regions [i].Dispose (); } regions.Clear (); rects.AddRange (groups_rects.toList ()); groups_rects.Dispose (); //Text Recognition (OCR) List detections = new List (); for (int i = 0; i < (int)rects.Count; i++) { Mat group_img = new Mat (); maskMat.submat (rects [i]).copyTo (group_img); Core.copyMakeBorder (group_img, group_img, 15, 15, 15, 15, Core.BORDER_CONSTANT, new Scalar (0)); detections.Add (group_img); } Debug.Log ("detections.Count " + detections.Count); //#Visualization for (int i = 0; i < rects.Count; i++) { Imgproc.rectangle (frame, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (255, 0, 0), 2); Imgproc.rectangle (frame, new Point (rects [i].x, rects [i].y), new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new Scalar (255, 255, 255), 1); string output = decoder.run (detections [i], 0); if (!string.IsNullOrEmpty (output)) { Debug.Log ("output " + output); Imgproc.putText (frame, output, new Point (rects [i].x, rects [i].y), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (0, 0, 255), 1, Imgproc.LINE_AA, false); } } Texture2D texture = new Texture2D (frame.cols (), frame.rows (), TextureFormat.RGBA32, false); Utils.matToTexture2D (frame, texture); // Texture2D texture = new Texture2D (detections [0].cols (), detections [0].rows (), TextureFormat.RGBA32, false); // // Utils.matToTexture2D (detections [0], texture); gameObject.GetComponent ().material.mainTexture = texture; for (int i = 0; i < detections.Count; i++) { detections [i].Dispose (); } binaryMat.Dispose (); maskMat.Dispose (); Utils.setDebugMode (false); } // Update is called once per frame void Update () { } /// /// Gets the transition probabilities data. /// /// The transition probabilities data. /// File path. double[] GetTransitionProbabilitiesData (string filePath) { XmlDocument xmlDoc = new XmlDocument (); xmlDoc.Load (filePath); XmlNode dataNode = xmlDoc.GetElementsByTagName ("data").Item (0); // Debug.Log ("dataNode.InnerText " + dataNode.InnerText); string[] dataString = dataNode.InnerText.Split (new string[] { " ", "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries); // Debug.Log ("dataString.Length " + dataString.Length); double[] data = new double[dataString.Length]; for (int i = 0; i < data.Length; i++) { try { data [i] = Convert.ToDouble (dataString [i]); } catch (FormatException) { Debug.Log ("Unable to convert '{" + dataString [i] + "}' to a Double."); } catch (OverflowException) { Debug.Log ("'{" + dataString [i] + "}' is outside the range of a Double."); } } return data; } /// /// Raises the destroy event. /// void OnDestroy () { #if UNITY_WEBGL && !UNITY_EDITOR if (getFilePath_Coroutine != null) { StopCoroutine (getFilePath_Coroutine); ((IDisposable)getFilePath_Coroutine).Dispose (); } #endif } /// /// Raises the back button click event. /// public void OnBackButtonClick () { SceneManager.LoadScene ("OpenCVForUnityExample"); } } } #endif