arskrald/AR-3/Assets/python_test.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using OpenCVForUnity.CoreModule;
using UnityEngine;
using Vuforia;
using Newtonsoft.Json;
using RestSharp;
using OpenCVForUnity.Calib3dModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.ImgcodecsModule;
using System.Collections;

public class HomographyAndState
{
    public List<List<float>> homography { get; set; }
    public List<List<int>> board { get; set; }
}

public class python_test : MonoBehaviour
{
    Mat camImageMat;
    Texture2D outputTexture;
    int baselineWidth = 1500;
    int baselineHeight = 1500;
    Mat loadedImage;
    GameObject rue;
    bool displayWarped = false;
    double rueX = 0.51;
    double rueY = 0.71;

    Mat camImgCopy;

    WebCamTexture _webcamtex;
    Mat camMatFromTexture;

    Process process;
    ProcessStartInfo start = new ProcessStartInfo();
    Texture2D _TextureFromCamera;

    RestClient restClient;

    int viewMarked = 0;
    //StreamWriter sw;
    //StreamReader sr;


    // Start is called before the first frame update
    void Start()
    {

        //loadedImage = Imgcodecs.imread("/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/whole_boards/boards_for_empty/board_1554285984.187497_rank_4.png");
        //camImgCopy = loadedImage.clone();

        start.FileName = "/Users/alexandermunch-hansen/.virtualenvs/cv/bin/python3";
        var cmd = "/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/adapter.py";
        var args = "";
        start.Arguments = string.Format("\"{0}\" \"{1}\"", cmd, args);
        print(start.Arguments);
        start.UseShellExecute = false;// Do not use OS shell
        start.CreateNoWindow = true; // We don't need new window
        start.RedirectStandardInput = true;
        start.RedirectStandardOutput = true;// Any output, generated by application will be redirected back
        start.RedirectStandardError = true; // Any error in standard output will be redirected back (for example exceptions)


        camMatFromTexture = new Mat(1080, 1920, CvType.CV_8UC3);
        _TextureFromCamera = new Texture2D(1920, 1080);

        restClient = new RestClient("http://localhost:5000");


        //process = Process.Start(start);
        //sw = new StreamWriter(process.StandardInput.BaseStream, System.Text.Encoding.Default, 1024, true);
        //sr = new StreamReader(process.StandardOutput.BaseStream, System.Text.Encoding.Default, true, 1024, true);

        _webcamtex = new WebCamTexture(WebCamTexture.devices[0].name, 1920, 1080);
        _webcamtex.Play();



    }



    string runCmd(Mat image)
    {
        var buffer = new MatOfByte();
        OpenCVForUnity.ImgcodecsModule.Imgcodecs.imencode(".png", image, buffer);
        string base64 = Convert.ToBase64String(buffer.toArray());

        String json = $"{{\"img\": \"{base64}\"}}";

        RestRequest request = new RestRequest(Method.POST);

        request.RequestFormat = DataFormat.Json;

        request.AddParameter("application/json", json, ParameterType.RequestBody);
        var response = restClient.Execute(request);

        return response.Content;
    }

    /*
    using (StreamWriter writer = process.StandardInput)
    {
        writer.WriteLine(base64);
    }


    using (StreamReader reader = process.StandardOutput)
    {
        string result = reader.ReadLine();
        return (string)result;

    }*/



    // Update is called once per frame
    void Update()
    {
        Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888);

        Mat textMat = null;

        if (camImg != null)
        {
            _TextureFromCamera.SetPixels(_webcamtex.GetPixels());
            _TextureFromCamera.Apply();
            //byte[] bytes = _TextureFromCamera.EncodeToPNG();
            //string filePath = "SavedScreen1.png";
            //File.WriteAllBytes(filePath, bytes);
            OpenCVForUnity.UnityUtils.Utils.texture2DToMat(_TextureFromCamera, camMatFromTexture);



            /*
            if (camImageMat == null)
            {
                camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4);
            }
            camImageMat.put(0, 0, camImg.Pixels);
            */


            if (Input.GetKeyDown(KeyCode.A))
            {
                print("fuck");
                viewMarked = 0;
            } else if (Input.GetKeyDown(KeyCode.S))
            {
                print("cock");
                viewMarked = 1;
            }


            if (Input.GetKeyDown("space"))
            {
                //ScreenCapture.CaptureScreenshot("plspls.png");
                
                loadedImage = camMatFromTexture.clone();

                // TODO: We should clone camImageMat again, instead of the loadedImage I use now

                //loadedImage = camImageMat.clone();
                //camImgCopy = camImageMat.clone();

                print("------------------------------------");

                //print(loadedImage.size());
                camImageMat = loadedImage.clone();

                print("------------------------------------");

     
                var pls = runCmd(camMatFromTexture);
                

                Mat homography = new Mat(new Size(3, 3), CvType.CV_64F);



                HomographyAndState homographyAndState = JsonConvert.DeserializeObject<HomographyAndState>(pls);


                for (int i = 0; i < 3; i++)
                {
                    for (int j = 0; j < 3; j++)
                    {
                        homography.put(i, j, homographyAndState.homography[i][j]);
                    }
                }

                MatOfPoint2f points = new MatOfPoint2f();
                points.alloc(64);

                var board = homographyAndState.board;
                int indexer = 0;
                for (int y = 1; y < 9; y++)
                {
                    for (int x = 1; x < 9; x++)
                    {
                        var cur_pos = board[x - 1][y - 1];

                        points.put(indexer, 0, (cur_pos != 6 ? 1 : -200) * y * 165, (cur_pos != 6 ? 1 : -200) * x * 165);
                        indexer += 1;
                    }
                }

                MatOfPoint2f dstPoints = new MatOfPoint2f();
                
                Core.perspectiveTransform(points, dstPoints, homography.inv());

                List<Point> transformedPoints = dstPoints.toList();

                foreach (Point point in transformedPoints)
                {
                    Imgproc.circle(loadedImage, point, 10, new Scalar(255, 0, 0), -1);
                }

                // MatDisplay.DisplayMat(loadedImage, MatDisplaySettings.FULL_BACKGROUND);
            
                //process.Close();
                //process = Process.Start(start);

            }

            if (viewMarked == 0)
            {
                MatDisplay.DisplayMat(camMatFromTexture, MatDisplaySettings.FULL_BACKGROUND);

            } else if (viewMarked == 1)
            {
                MatDisplay.DisplayMat(loadedImage, MatDisplaySettings.FULL_BACKGROUND);
            }


        }
    }


}