From be7dc71647e0c339bf804fde5cddd61de6beb1a2 Mon Sep 17 00:00:00 2001 From: Alexander Munch-Hansen Date: Tue, 23 Apr 2019 12:53:37 +0200 Subject: [PATCH] inverse homo stuff --- AR-3/Assets/python_test.cs | 185 +++++++++++++++++++++++-------------- 1 file changed, 116 insertions(+), 69 deletions(-) diff --git a/AR-3/Assets/python_test.cs b/AR-3/Assets/python_test.cs index b5b9821..7c97ef2 100644 --- a/AR-3/Assets/python_test.cs +++ b/AR-3/Assets/python_test.cs @@ -1,21 +1,21 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using OpenCVForUnity.CoreModule; +using System.Diagnostics; +using System.IO; +using OpenCVForUnity.CoreModule; using UnityEngine; using Vuforia; using Newtonsoft.Json; using OpenCVForUnity.Calib3dModule; using OpenCVForUnity.ImgprocModule; -using OpenCVForUnity.ImgcodecsModule; - -public class HomographyPoints +using OpenCVForUnity.ImgcodecsModule; + +public class HomographyAndState { - public List> src_points { get; set; } - public List> dst_points { get; set; } + public List> homography { get; set; } + public List> board { get; set; } } - + public class python_test : MonoBehaviour { Mat camImageMat; @@ -23,28 +23,38 @@ public class python_test : MonoBehaviour int baselineWidth = 1500; int baselineHeight = 1500; Mat loadedImage; + GameObject rue; + bool displayWarped = false; + double rueX = 0.51; + double rueY = 0.71; + + Mat camImgCopy; + + // Start is called before the first frame update void Start() { - loadedImage = Imgcodecs.imread("/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/whole_boards/boards_for_empty/board_1554285984.187497_rank_4.png"); + loadedImage = Imgcodecs.imread("/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/whole_boards/boards_for_empty/board_1554285984.187497_rank_4.png"); + camImgCopy = loadedImage.clone(); + } - string runCmd(Mat image) - { - var buffer = new MatOfByte(); + string runCmd(Mat image) + { + var buffer = new MatOfByte(); OpenCVForUnity.ImgcodecsModule.Imgcodecs.imencode(".png", camImageMat, buffer); string base64 = Convert.ToBase64String(buffer.toArray()); - ProcessStartInfo start = new ProcessStartInfo(); start.FileName = "/Users/alexandermunch-hansen/.virtualenvs/cv/bin/python3"; var cmd = "/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/adapter.py"; var args = ""; start.Arguments = string.Format("\"{0}\" \"{1}\"", cmd, args); + print(start.Arguments); start.UseShellExecute = false;// Do not use OS shell start.CreateNoWindow = true; // We don't need new window start.RedirectStandardInput = true; @@ -52,16 +62,15 @@ public class python_test : MonoBehaviour start.RedirectStandardError = true; // Any error in standard output will be redirected back (for example exceptions) using (Process process = Process.Start(start)) { - using (StreamWriter writer = process.StandardInput) - { - writer.WriteLine(base64); - writer.Close(); - } + - process.WaitForExit(); - - string err = process.StandardError.ReadToEnd(); - print(err); + using (StreamWriter writer = process.StandardInput) + { + writer.WriteLine(base64); + writer.Close(); + } + + process.WaitForExit(); using (StreamReader reader = process.StandardOutput) { @@ -69,22 +78,22 @@ public class python_test : MonoBehaviour return (string) result; } - } + } } // Update is called once per frame void Update() - { - Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888); + { + Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888); Mat outputMat = null; if (camImg != null) - { - if (camImageMat == null) - { - camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4); - } + { + if (camImageMat == null) + { + camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4); + } camImageMat.put(0, 0, camImg.Pixels); - if (Input.GetKeyDown("space")) + if (Input.GetKeyDown("space")) { /* @@ -95,53 +104,91 @@ public class python_test : MonoBehaviour print("------------------------------------"); print(loadedImage.size()); - var camImgCopy = loadedImage.clone(); outputMat = camImgCopy.clone(); - camImageMat = loadedImage.clone(); - - print("------------------------------------"); - + camImageMat = loadedImage.clone(); + + print("------------------------------------"); + var pls = runCmd(camImageMat); - print(pls); - - HomographyPoints homographyPoints = JsonConvert.DeserializeObject(pls); - - print(homographyPoints.src_points); - - MatOfPoint2f imagePoints = new MatOfPoint2f(); - imagePoints.alloc(homographyPoints.src_points.Count); - print(homographyPoints.src_points.Count); - - - MatOfPoint2f dstPoints = new MatOfPoint2f(); - dstPoints.alloc(homographyPoints.dst_points.Count); + Mat homography = new Mat(new Size(3, 3), CvType.CV_64F); - for (int i = 0; i < homographyPoints.src_points.Count; i++) - { - imagePoints.put(i, 0, (int) homographyPoints.src_points[i][0], (int) homographyPoints.src_points[i][1]); - } - - for (int i = 0; i < homographyPoints.dst_points.Count; i++) + + HomographyAndState homographyAndState = JsonConvert.DeserializeObject(pls); + + print(homographyAndState.homography); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) + { + homography.put(i, j, homographyAndState.homography[i][j]); + print(homographyAndState.homography[i][j]); + } + } + - dstPoints.put(i, 0, (int) homographyPoints.dst_points[i][0], (int) homographyPoints.dst_points[i][1]); - } - - var homography = Calib3d.findHomography(imagePoints, dstPoints, Calib3d.RANSAC); - Imgproc.warpPerspective(camImgCopy, outputMat, homography, new Size(baselineWidth, baselineHeight)); - imagePoints.Dispose(); - dstPoints.Dispose(); - - } + foreach (var row in homographyAndState.board) { + foreach (var lel in row) { + print(lel); + } + } - if (outputMat != null) - MatDisplay.DisplayMat(outputMat, MatDisplaySettings.FULL_BACKGROUND); - } - } + + //List points = new List(); + MatOfPoint2f points = new MatOfPoint2f(); + points.alloc(16); + + int cur_level = 0; + int index = 0; + for (int x = 0; x < 8; x++) + { + cur_level += 165; + foreach (int y in new List { 150, 315 }) + { + + //points.put(index, 0, cur_level, y); + //index += 1; + Imgproc.circle(outputMat, new Point(cur_level, y), 50, new Scalar(255, 0, 0), -1); + } + } + + //Core.perspectiveTransform(points, transformedPoints, homography.inv()); + + //for (int i = 0; i < 16; i++) + //{ + // Imgproc.circle(loadedImage, transformedPoints.get(i, 0), 5, new Scalar(255, 0, 0), -1); + //} + + + + if (displayWarped) + { + camImgCopy = loadedImage.clone(); + Mat tmpCopy = camImgCopy.clone(); + Imgproc.warpPerspective(outputMat,tmpCopy, homography.inv(), camImgCopy.size()); + Core.addWeighted(tmpCopy, 0.80f, camImgCopy, 0.7f, 0.0f, camImgCopy); + displayWarped = false; + } + else + { + displayWarped = true; + MatDisplay.DisplayMat(outputMat, MatDisplaySettings.FULL_BACKGROUND); + + } + + } + + if (!displayWarped) + MatDisplay.DisplayMat(camImgCopy, MatDisplaySettings.FULL_BACKGROUND); + + + } + } + }