arskrald/AR-3/Assets/python_test.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using OpenCVForUnity.CoreModule;
using UnityEngine;
using Vuforia;
using Newtonsoft.Json;
using OpenCVForUnity.Calib3dModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.ImgcodecsModule;

public class HomographyAndState
{
    public List<List<float>> homography { get; set; }
    public List<List<int>> board { get; set; }
}

public class python_test : MonoBehaviour
{
    Mat camImageMat;
    Texture2D outputTexture;
    int baselineWidth = 1500;
    int baselineHeight = 1500;
    Mat loadedImage;
    GameObject rue;
    bool displayWarped = false;
    double rueX = 0.51;
    double rueY = 0.71;

    Mat camImgCopy;



    // Start is called before the first frame update
    void Start()
    {

        loadedImage = Imgcodecs.imread("/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/whole_boards/boards_for_empty/board_1554285984.187497_rank_4.png");
        camImgCopy = loadedImage.clone();


    }


    string runCmd(Mat image)
    {
        var buffer = new MatOfByte();
        OpenCVForUnity.ImgcodecsModule.Imgcodecs.imencode(".png", camImageMat, buffer);
        string base64 = Convert.ToBase64String(buffer.toArray());

        ProcessStartInfo start = new ProcessStartInfo();
        start.FileName = "/Users/alexandermunch-hansen/.virtualenvs/cv/bin/python3";
        var cmd = "/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/adapter.py";
        var args = "";
        start.Arguments = string.Format("\"{0}\" \"{1}\"", cmd, args);
        print(start.Arguments);
        start.UseShellExecute = false;// Do not use OS shell
        start.CreateNoWindow = true; // We don't need new window
        start.RedirectStandardInput = true;
        start.RedirectStandardOutput = true;// Any output, generated by application will be redirected back
        start.RedirectStandardError = true; // Any error in standard output will be redirected back (for example exceptions)
        using (Process process = Process.Start(start))
        {
 

            using (StreamWriter writer = process.StandardInput)
            {
                writer.WriteLine(base64);
                writer.Close();
            }

            process.WaitForExit();

            using (StreamReader reader = process.StandardOutput)
            {
                string result = reader.ReadLine();

                return (string) result;
            }
        }
    }

    // Update is called once per frame
    void Update()
    {
        Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888);
        Mat outputMat = null;
        if (camImg != null)
        {
            if (camImageMat == null)
            {
                camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4);
            }
            camImageMat.put(0, 0, camImg.Pixels);
            if (Input.GetKeyDown("space"))
            {

                /*
                var camImgCopy = camImageMat.clone();
                outputMat = camImageMat.clone();
                */

                print("------------------------------------");
                print(loadedImage.size());

                outputMat = camImgCopy.clone();
                camImageMat = loadedImage.clone();

                print("------------------------------------");

                var pls = runCmd(camImageMat);

                Mat homography = new Mat(new Size(3, 3), CvType.CV_64F);



                HomographyAndState homographyAndState = JsonConvert.DeserializeObject<HomographyAndState>(pls);

                print(homographyAndState.homography);

                for (int i = 0; i < 3; i++)
                {
                    for (int j = 0; j < 3; j++)
                    {
                        homography.put(i, j, homographyAndState.homography[i][j]);
                        print(homographyAndState.homography[i][j]);
                    }
                }


                Imgproc.warpPerspective(camImgCopy, outputMat, homography, new Size(baselineWidth, baselineHeight));


                //List<Point> points = new List<Point>();
                MatOfPoint2f points = new MatOfPoint2f();
                points.alloc(64);

                var board = homographyAndState.board;
                int indexer = 0;
                for (int y = 1; y < 9; y++)
                {
                    for (int x = 1; x < 9; x++)
                    {
                        var cur_pos = board[x - 1][y - 1];
                        points.put(indexer, 0, (cur_pos != 6 ? 1 : 0) * y * 165, (cur_pos != 6 ? 1 : 0) * x * 165);
                        indexer += 1;
                    }
                }


                /*
                int cur_level = 0;
                int index = 0;
                for (int x = 0; x < 8; x++)
                {
                    cur_level += 165;
                    foreach (int y in new List<int> { 150, 315 })
                    {

                        points.put(index, 0, cur_level, y);
                        index += 1;
                        //Imgproc.circle(outputMat, new Point(cur_level, y), 50, new Scalar(255, 0, 0), -1);
                    }
                }*/

                MatOfPoint2f dstPoints = new MatOfPoint2f();
                
                Core.perspectiveTransform(points, dstPoints, homography.inv());

                List<Point> transformedPoints = dstPoints.toList();

                foreach (Point point in transformedPoints)
                {
                    Imgproc.circle(loadedImage, point, 10, new Scalar(255, 0, 0), -1);
                }

                MatDisplay.DisplayMat(loadedImage, MatDisplaySettings.FULL_BACKGROUND);
                displayWarped = true;

                /*
                if (displayWarped)
                {
                    camImgCopy = loadedImage.clone();
                    Mat tmpCopy = camImgCopy.clone();
                    Imgproc.warpPerspective(outputMat,tmpCopy, homography.inv(), camImgCopy.size());
                    Core.addWeighted(tmpCopy, 0.80f, camImgCopy, 0.7f, 0.0f, camImgCopy);
                    displayWarped = false;
                }
                else
                {
                    displayWarped = true;
                    MatDisplay.DisplayMat(outputMat, MatDisplaySettings.FULL_BACKGROUND);

                }
                */               

            }

            if (!displayWarped)
                MatDisplay.DisplayMat(camImgCopy, MatDisplaySettings.FULL_BACKGROUND);


        }
    }


}