arskrald/AR-3/Assets/python_test.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using OpenCVForUnity.CoreModule;
using UnityEngine;
using Vuforia;
using Newtonsoft.Json;
using RestSharp;
using OpenCVForUnity.Calib3dModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.ImgcodecsModule;
using System.Collections;

public class HomographyAndState
{
    public List<List<float>> homography { get; set; }
    public List<List<int>> board { get; set; }
}

public class python_test : MonoBehaviour
{
    Mat camImageMat;
    Texture2D outputTexture;
    int baselineWidth = 1500;
    int baselineHeight = 1500;
    Mat loadedImage;
    Mat loadedImage_pls;
    GameObject rue;
    bool displayWarped = false;
    double rueX = 0.51;
    double rueY = 0.71;

    Mat camImgCopy;

    WebCamTexture _webcamtex;
    Mat camMatFromTexture;

    Process process;
    ProcessStartInfo start = new ProcessStartInfo();
    Texture2D _TextureFromCamera;

    bool displayLoaded = false;
    RestClient restClient;

    Dictionary<int, string> intToPiece = new Dictionary<int, string>();


    int viewMarked = 0;
    //StreamWriter sw;
    //StreamReader sr;


    // Start is called before the first frame update
    void Start()
    {




        loadedImage_pls = Imgcodecs.imread("/Users/alexandermunch-hansen/projects/chess_stuff/advancedskrald/whole_boards/boards_for_empty/board_1554285984.187497_rank_4.png");


        intToPiece.Add(0, "RK");
        intToPiece.Add(1, "KNT");
        intToPiece.Add(2, "BSP");
        intToPiece.Add(6, "");


        camMatFromTexture = new Mat(1080, 1920, CvType.CV_8UC3);
        _TextureFromCamera = new Texture2D(1920, 1080);




        restClient = new RestClient("http://localhost:5000");


        //process = Process.Start(start);
        //sw = new StreamWriter(process.StandardInput.BaseStream, System.Text.Encoding.Default, 1024, true);
        //sr = new StreamReader(process.StandardOutput.BaseStream, System.Text.Encoding.Default, true, 1024, true);

        _webcamtex = new WebCamTexture(WebCamTexture.devices[0].name, 1920, 1080);
        _webcamtex.Play();



    }



    string runCmd(Mat image)
    {
        var buffer = new MatOfByte();
        OpenCVForUnity.ImgcodecsModule.Imgcodecs.imencode(".png", image, buffer);
        string base64 = Convert.ToBase64String(buffer.toArray());

        String json = $"{{\"img\": \"{base64}\"}}";

        RestRequest request = new RestRequest(Method.POST);

        request.RequestFormat = DataFormat.Json;

        request.AddParameter("application/json", json, ParameterType.RequestBody);
        var response = restClient.Execute(request);
        print(response.Content);
        return response.Content;
    }

    /*
    using (StreamWriter writer = process.StandardInput)
    {
        writer.WriteLine(base64);
    }


    using (StreamReader reader = process.StandardOutput)
    {
        string result = reader.ReadLine();
        return (string)result;

    }*/



    // Update is called once per frame
    void Update()
    {
        if (displayLoaded)
        {
            camMatFromTexture = loadedImage_pls.clone();
        }


        Image camImg = CameraDevice.Instance.GetCameraImage(Image.PIXEL_FORMAT.RGBA8888);

        Mat textMat = null;

        if (camImg != null)
        {
            _TextureFromCamera.SetPixels(_webcamtex.GetPixels());
            _TextureFromCamera.Apply();
            //byte[] bytes = _TextureFromCamera.EncodeToPNG();
            //string filePath = "SavedScreen1.png";
            //File.WriteAllBytes(filePath, bytes);
            if (!displayLoaded)
            {
                OpenCVForUnity.UnityUtils.Utils.texture2DToMat(_TextureFromCamera, camMatFromTexture);
            }


            /*
            if (camImageMat == null)
            {
                camImageMat = new Mat(camImg.Height, camImg.Width, CvType.CV_8UC4);
            }
            camImageMat.put(0, 0, camImg.Pixels);
            */


            if (Input.GetKeyDown(KeyCode.A))
            {
                viewMarked = 0;
            } else if (Input.GetKeyDown(KeyCode.S))
            {
                viewMarked = 1;
            } else if (Input.GetKeyDown(KeyCode.D))
            {
                displayLoaded = true;
            } else if (Input.GetKeyDown(KeyCode.F))
            {
                displayLoaded = false;
            }


            if (Input.GetKeyDown("space"))
            {
                //ScreenCapture.CaptureScreenshot("plspls.png");

                // TODO: Set this back again

                if (!displayLoaded)
                {
                    loadedImage = camMatFromTexture.clone();
                }
                else
                {
                    loadedImage = loadedImage_pls.clone();
                }
                //camMatFromTexture = loadedImage.clone();



                var pls = runCmd(camMatFromTexture);
                

                Mat homography = new Mat(new Size(3, 3), CvType.CV_64F);



                HomographyAndState homographyAndState = JsonConvert.DeserializeObject<HomographyAndState>(pls);


                for (int i = 0; i < 3; i++)
                {
                    for (int j = 0; j < 3; j++)
                    {
                        homography.put(i, j, homographyAndState.homography[i][j]);
                    }
                }

                MatOfPoint2f points = new MatOfPoint2f();
                points.alloc(64);


                var board = homographyAndState.board;
                int indexer = 0;
                for (int y = 1; y < 9; y++)
                {
                    for (int x = 1; x < 9; x++)
                    {
                        var cur_pos = board[x - 1][y - 1];


                        points.put(indexer, 0, (cur_pos != 6 ? 1 : -200) * y * 165, (cur_pos != 6 ? 1 : -200) * x * 165);
                        indexer += 1;
                    }
                }

                MatOfPoint2f dstPoints = new MatOfPoint2f();
                
                Core.perspectiveTransform(points, dstPoints, homography.inv());

                List<Point> transformedPoints = dstPoints.toList();

                indexer = 0;
                for (int i = 0; i < 8; i++)
                {
                    for (int j = 0; j < 8; j++) {
                        var point = transformedPoints[indexer];
                        //Imgproc.circle(loadedImage, point, 5, new Scalar(255, 255, 255), -1);
                        print($"({point.x}, {point.y}): {intToPiece[board[j][i]]}");
                        
                        Imgproc.putText(loadedImage, $"{intToPiece[board[j][i]]}", new Point(point.x-30, point.y - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1, new Scalar(255, 255, 255), 3);
                        indexer += 1;
                        }
                }
    

                // MatDisplay.DisplayMat(loadedImage, MatDisplaySettings.FULL_BACKGROUND);
            
                //process.Close();
                //process = Process.Start(start);

            }

            if (viewMarked == 0)
            {
                MatDisplay.DisplayMat(camMatFromTexture, MatDisplaySettings.FULL_BACKGROUND);

            } else if (viewMarked == 1)
            {
                MatDisplay.DisplayMat(loadedImage, MatDisplaySettings.FULL_BACKGROUND);
            }


        }
    }


}