From 28361633547d2d4e2db7e5aebb1d26f35200c567 Mon Sep 17 00:00:00 2001 From: Alexander Munch-Hansen Date: Tue, 5 Mar 2019 15:58:23 +0100 Subject: [PATCH] fix --- AR-2b/Assets/NewTestMaterial.mat | 77 ++++++++++ AR-2b/Assets/detection_script.cs | 160 ++++++++++++++++++-- AR-2b/Assets/detection_script_2.cs | 228 ----------------------------- 3 files changed, 223 insertions(+), 242 deletions(-) create mode 100644 AR-2b/Assets/NewTestMaterial.mat delete mode 100644 AR-2b/Assets/detection_script_2.cs diff --git a/AR-2b/Assets/NewTestMaterial.mat b/AR-2b/Assets/NewTestMaterial.mat new file mode 100644 index 0000000..ca972ce --- /dev/null +++ b/AR-2b/Assets/NewTestMaterial.mat @@ -0,0 +1,77 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!21 &2100000 +Material: + serializedVersion: 6 + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_Name: NewTestMaterial + m_Shader: {fileID: 46, guid: 0000000000000000f000000000000000, type: 0} + m_ShaderKeywords: + m_LightmapFlags: 4 + m_EnableInstancingVariants: 0 + m_DoubleSidedGI: 0 + m_CustomRenderQueue: -1 + stringTagMap: {} + disabledShaderPasses: [] + m_SavedProperties: + serializedVersion: 3 + m_TexEnvs: + - _BumpMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _DetailAlbedoMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _DetailMask: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _DetailNormalMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _EmissionMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _MainTex: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _MetallicGlossMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _OcclusionMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + - _ParallaxMap: + m_Texture: {fileID: 0} + m_Scale: {x: 1, y: 1} + m_Offset: {x: 0, y: 0} + m_Floats: + - _BumpScale: 1 + - _Cutoff: 0.5 + - _DetailNormalMapScale: 1 + - _DstBlend: 0 + - _GlossMapScale: 1 + - _Glossiness: 0.5 + - _GlossyReflections: 1 + - _Metallic: 0 + - _Mode: 0 + - _OcclusionStrength: 1 + - _Parallax: 0.02 + - _SmoothnessTextureChannel: 0 + - _SpecularHighlights: 1 + - _SrcBlend: 1 + - _UVSec: 0 + - _ZWrite: 1 + m_Colors: + - _Color: {r: 1, g: 1, b: 1, a: 1} + - _EmissionColor: {r: 0, g: 0, b: 0, a: 1} diff --git a/AR-2b/Assets/detection_script.cs b/AR-2b/Assets/detection_script.cs index 9816858..71fb5f2 100644 --- a/AR-2b/Assets/detection_script.cs +++ b/AR-2b/Assets/detection_script.cs @@ -6,29 +6,52 @@ using OpenCVForUnity.ImgprocModule; using System.Linq; using Vuforia; -public class detection_script_2 : MonoBehaviour +public class detection_script : MonoBehaviour { Mat cameraImageMat; Mat stylizedMat = new Mat(); Mat greyMat = new Mat(); - int width = 400; - int height = 400; + int width = 1050; + int height = 1050; Texture2D outputTexture; MatOfPoint2f dstPoints; + MatOfPoint2f dstPointsInv; + + + Mat skullTextureMat; + + + private GameObject pls; + private Renderer rend; // Start is called before the first frame update void Start() { + skullTextureMat = MatDisplay.LoadRGBATexture("Resources/flying_skull_tex.png"); + dstPoints = new MatOfPoint2f(); dstPoints.alloc(4); - dstPoints.put(3, 0, width, height); + /*dstPoints.put(3, 0, width, height); dstPoints.put(2, 0, 0, height); dstPoints.put(1, 0, width, 0); - dstPoints.put(0, 0, 0, 0); + dstPoints.put(0, 0, 0, 0);*/ + + + dstPoints.put(3, 0, 0, 0); + dstPoints.put(2, 0, width, 0); + dstPoints.put(0, 0, 0, height); + dstPoints.put(1, 0, width, height); + + + + outputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false); + + pls = GameObject.Find("flying_skull_001"); + rend = pls.GetComponent(); } // Update is called once per frame @@ -58,7 +81,7 @@ public class detection_script_2 : MonoBehaviour //Imgproc.drawContours(cameraImageMat, contourList, -1, new Scalar(255,0,0), 2); - List squareContours = new List(); + List squareContours = new List(); foreach (var contour in contourList) { MatOfPoint2f contour2f = new MatOfPoint2f(); @@ -70,13 +93,13 @@ public class detection_script_2 : MonoBehaviour if (approx.toList().Count == 4) { - Imgproc.drawContours(cameraImageMat, new List { contour }, -1, new Scalar(255,0,0), 2); - squareContours.Add(contour); + Imgproc.drawContours(cameraImageMat, new List { contour }, -1, new Scalar(255, 0, 0), 2); + squareContours.Add(approx); } } - MatOfPoint square = findSquare(squareContours); + MatOfPoint2f square = findSquare(squareContours); if (square == null) { MatDisplay.DisplayMat(cameraImageMat, MatDisplaySettings.FULL_BACKGROUND); @@ -84,16 +107,79 @@ public class detection_script_2 : MonoBehaviour return; } - Imgproc.drawContours(cameraImageMat, new List { square }, -1, new Scalar(0, 255, 0), 2); + for (int i = 0; i < 4; i++) + { + double x = square.get(i, 0)[0]; + double y = square.get(i, 0)[1]; + Imgproc.circle(cameraImageMat, new Point(x, y), 10, new Scalar(0, 255, 0, 255)); + + + } + + Mat homo = ComputeHomo(square, dstPoints); + + + Mat outputSkullMat = cameraImageMat.clone(); + //Mat outputSkullMat = skullTextureMat.clone(); + + Imgproc.warpPerspective(skullTextureMat, outputSkullMat, homo.inv(), outputSkullMat.size()); - MatDisplay.DisplayMat(cameraImageMat, MatDisplaySettings.FULL_BACKGROUND); - + + + + + Mat dstCam = cameraImageMat.clone(); + + + Core.addWeighted(cameraImageMat, 0.95f, outputSkullMat, 0.7f, 0.0f, dstCam); + + //Display the Mat that includes video feed and debug points + MatDisplay.DisplayMat(dstCam, MatDisplaySettings.FULL_BACKGROUND); + + + + + /* + Mat homo = new Mat(); + homo = ComputeHomo(square, dstPoints); + + Mat outputMat = cameraImageMat.clone(); + + Imgproc.warpPerspective(cameraImageMat, outputMat, homo, new Size(outputMat.width(), outputMat.height())); + + + Mat outputSkullMat = skullTextureMat.clone(); + + + + + var rectOutputMat = new Mat(outputMat, new OpenCVForUnity.CoreModule.Rect(0, 0, width, height)); + + MatDisplay.MatToTexture(rectOutputMat, ref outputTexture); + + + rend.sharedMaterial.mainTexture = outputTexture; + + */ + + + //MatDisplay.DisplayMat(cameraImageMat, MatDisplaySettings.FULL_BACKGROUND); + } } - MatOfPoint findSquare (List squareContours) + + + bool checkSize(double outer, double inner) + { + print(outer); + print(inner); + return (outer > inner && outer < (inner * 2)); + } + + MatOfPoint2f findSquare (List squareContours) { foreach (var outer_square in squareContours) { @@ -103,6 +189,9 @@ public class detection_script_2 : MonoBehaviour var outer_minX = outer_square.toList().Min(point => point.x); var outer_minY = outer_square.toList().Min(point => point.y); + var outer_size = (outer_maxX - outer_minX) * (outer_maxY - outer_minY); + + foreach (var inner_square in squareContours) { var inner_maxX = inner_square.toList().Max(point => point.x); @@ -111,15 +200,58 @@ public class detection_script_2 : MonoBehaviour var inner_minX = inner_square.toList().Min(point => point.x); var inner_minY = inner_square.toList().Min(point => point.y); + var inner_size = (inner_maxX - inner_minX) * (inner_maxY - inner_minY); + if (outer_minX < inner_minX && outer_minY < inner_minY && outer_maxX > inner_maxX && - outer_maxY > inner_maxY) + outer_maxY > inner_maxY && + checkSize(outer_size, inner_size)) { + return outer_square; } } } return null; } + + + Mat ComputeHomo(MatOfPoint2f imgPoints, MatOfPoint2f destPoints) + { + + + Mat H = new Mat(8, 1, CvType.CV_32FC1); + Mat A = new Mat(8, 8, CvType.CV_32FC1); + Mat b = new Mat(8, 1, CvType.CV_32FC1); + + + for (int i = 0; i < 4; i++) + { + var u = destPoints.get(i, 0)[0]; + var v = destPoints.get(i, 0)[1]; + + b.put(i * 2, 0, u); + b.put((i * 2) + 1, 0, v); + + + + var x = imgPoints.get(i, 0)[0]; + var y = imgPoints.get(i, 0)[1]; + + A.put(i * 2, 0, x, y, 1, 0, 0, 0, -u * x, -u * y); + A.put((i * 2) + 1, 0, 0, 0, 0, x, y, 1, -v * x, -v * y); + + } + + Core.solve(A, b, H); + + Mat ShitsReal = new Mat(3, 3, CvType.CV_32FC1); + ShitsReal.put(0, 0, H.get(0, 0)[0], H.get(1, 0)[0], H.get(2, 0)[0]); + ShitsReal.put(1, 0, H.get(3, 0)[0], H.get(4, 0)[0], H.get(5, 0)[0]); + ShitsReal.put(2, 0, H.get(6, 0)[0], H.get(7, 0)[0], 1); + + return ShitsReal; + } + } diff --git a/AR-2b/Assets/detection_script_2.cs b/AR-2b/Assets/detection_script_2.cs deleted file mode 100644 index 2645063..0000000 --- a/AR-2b/Assets/detection_script_2.cs +++ /dev/null @@ -1,228 +0,0 @@ -using System.Collections; -using System.Collections.Generic; -using UnityEngine; -using OpenCVForUnity.CoreModule; -using OpenCVForUnity.ImgprocModule; -using System.Linq; -using Vuforia; - -public class detection_script : MonoBehaviour -{ - Mat cameraImageMat; - Mat stylizedMat = new Mat(); - Mat greyMat = new Mat(); - - - int width = 100; - int height = 100; - - Texture2D outputTexture; - MatOfPoint2f dstPointsInv; - - MatOfPoint2f imagePoints; - - private Mat skullTextureMat; - - - - // Start is called before the first frame update - void Start() - { - skullTextureMat = MatDisplay.LoadRGBATexture("Resources/flying_skull_tex.png"); - - dstPointsInv = new MatOfPoint2f(); - dstPointsInv.alloc(4); - - dstPointsInv.put(2, 0, 0, height); - dstPointsInv.put(3, 0, width, height); - dstPointsInv.put(0, 0, 0, 0); - dstPointsInv.put(1, 0, width, 0); - - outputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false); - - imagePoints = new MatOfPoint2f(); - imagePoints.alloc(4); - } - - // Update is called once per frame - void Update() - { - MatDisplay.SetCameraFoV(41.5f); - - Image cameraImage = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888); - - if (cameraImage != null) - { - if (cameraImageMat == null) - { - cameraImageMat = new Mat(cameraImage.Height, cameraImage.Width, CvType.CV_8UC4); - } - cameraImageMat.put(0, 0, cameraImage.Pixels); - - Imgproc.cvtColor(cameraImageMat, greyMat, Imgproc.COLOR_RGB2GRAY); - - - Imgproc.threshold(greyMat, stylizedMat, 69, 255, Imgproc.THRESH_BINARY); - - - List contourList = new List(); - Mat hierarchy = new Mat(); - Imgproc.findContours(stylizedMat, contourList, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); - - //Imgproc.drawContours(cameraImageMat, contourList, -1, new Scalar(255,0,0), 2); - - List squareContours = new List(); - foreach (var contour in contourList) - { - MatOfPoint2f contour2f = new MatOfPoint2f(); - contour.convertTo(contour2f, CvType.CV_32FC2); - - double epsilon = 0.01f * Imgproc.arcLength(contour2f, true); - MatOfPoint2f approx = new MatOfPoint2f(); - Imgproc.approxPolyDP(contour2f, approx, epsilon, true); - - if (approx.toList().Count == 4) - { - Imgproc.drawContours(cameraImageMat, new List { contour }, -1, new Scalar(255,0,0), 2); - squareContours.Add(approx); - } - } - - - MatOfPoint2f square = findSquare(squareContours); - MatOfPoint greenSquare = new MatOfPoint(); - square.convertTo(greenSquare, CvType.CV_32S); - Imgproc.drawContours(cameraImageMat, new List { greenSquare }, -1, new Scalar(0, 255, 0), 2); - MatDisplay.DisplayMat(cameraImageMat, MatDisplaySettings.FULL_BACKGROUND); - - return; - - if (square != null) - { - - for (int i = 0; i < 4; i++) - { - print($"Square: {square.get(i, 0)}, idx {i}"); - imagePoints.put(i, 0, square.get(i, 0)[0], square.get(i, 0)[1]); - } - - //Debug draw points using OpenCV's drawing functions - Point imgPnt1 = new Point(imagePoints.get(0, 0)); - Point imgPnt2 = new Point(imagePoints.get(1, 0)); - Point imgPnt3 = new Point(imagePoints.get(2, 0)); - Point imgPnt4 = new Point(imagePoints.get(3, 0)); - Mat camImgCopy = cameraImageMat.clone(); - Mat outputMat = camImgCopy.clone(); - - - - Imgproc.circle(cameraImageMat, imgPnt1, 5, new Scalar(255, 0, 0, 255)); - Imgproc.circle(cameraImageMat, imgPnt2, 5, new Scalar(0, 255, 0, 255)); - Imgproc.circle(cameraImageMat, imgPnt3, 5, new Scalar(0, 0, 255, 255)); - Imgproc.circle(cameraImageMat, imgPnt4, 5, new Scalar(255, 255, 0, 255)); - - - //Imgproc.drawContours(cameraImageMat, new List { square }, -1, new Scalar(0, 255, 0), 2); - - - Mat homo = ComputeHomo(imagePoints, dstPointsInv); - - - Mat outputSkullMat = cameraImageMat.clone(); - Imgproc.warpPerspective(skullTextureMat, outputSkullMat, homo.inv(), outputSkullMat.size()); - - - Mat dstCam = cameraImageMat.clone(); - - Core.addWeighted(cameraImageMat, 0.95f, outputSkullMat, 0.7f, 0.0f, dstCam); - - //Display the Mat that includes video feed and debug points - MatDisplay.DisplayMat(dstCam, MatDisplaySettings.FULL_BACKGROUND); - - } - else - { - MatDisplay.DisplayMat(cameraImageMat, MatDisplaySettings.FULL_BACKGROUND); - } - - } - } - - MatOfPoint2f findSquare (List squareContours) - { - foreach (var outer_square in squareContours) - { - var outer_maxX = outer_square.toList().Max(point => point.x); - var outer_maxY = outer_square.toList().Max(point => point.y); - - var outer_minX = outer_square.toList().Min(point => point.x); - var outer_minY = outer_square.toList().Min(point => point.y); - - foreach (var inner_square in squareContours) - { - var inner_maxX = inner_square.toList().Max(point => point.x); - var inner_maxY = inner_square.toList().Max(point => point.y); - - var inner_minX = inner_square.toList().Min(point => point.x); - var inner_minY = inner_square.toList().Min(point => point.y); - - if (outer_minX < inner_minX && - outer_minY < inner_minY && - outer_maxX > inner_maxX && - outer_maxY > inner_maxY) - { - - List sortedOuterSquare = outer_square.toList().OrderBy(p => p.x).ThenBy(p => p.y).ToList(); - - MatOfPoint2f res = new MatOfPoint2f(); - - for (int i = 0; i < 4; i++) - { - res.put(i, 0, sortedOuterSquare[i].x, sortedOuterSquare[i].y); - } - - print("--------------------------------"); - return res; - } - } - } - return null; - } - - Mat ComputeHomo(MatOfPoint2f imgPoints, MatOfPoint2f destPoints) - { - - - Mat H = new Mat(8, 1, CvType.CV_32FC1); - Mat A = new Mat(8, 8, CvType.CV_32FC1); - Mat b = new Mat(8, 1, CvType.CV_32FC1); - - - for (int i = 0; i < 4; i++) - { - var u = destPoints.get(i, 0)[0]; - var v = destPoints.get(i, 0)[1]; - - b.put(i * 2, 0, u); - b.put((i * 2) + 1, 0, v); - - - - var x = imgPoints.get(i, 0)[0]; - var y = imgPoints.get(i, 0)[1]; - - A.put(i * 2, 0, x, y, 1, 0, 0, 0, -u * x, -u * y); - A.put((i * 2) + 1, 0, 0, 0, 0, x, y, 1, -v * x, -v * y); - - } - - Core.solve(A, b, H); - - Mat ShitsReal = new Mat(3, 3, CvType.CV_32FC1); - ShitsReal.put(0, 0, H.get(0, 0)[0], H.get(1, 0)[0], H.get(2, 0)[0]); - ShitsReal.put(1, 0, H.get(3, 0)[0], H.get(4, 0)[0], H.get(5, 0)[0]); - ShitsReal.put(2, 0, H.get(6, 0)[0], H.get(7, 0)[0], 1); - - return ShitsReal; - } -}