Step 3: Moving the particles based on HPE key points
In this step, I try to move the spheres on Unity based on the key points which are come from the MQTT message. For changing the position a game object I will use the transform.position code. In the python code which is run.py (image body estimation) I change the output of the pose_3d because it separates the value x,y,z but I needed the separate the key points value therefore I took the transpose of pose_3d
|
1 |
pose_3dqt = np.array(pose_3d[0]).transpose() |
This gave me the ınduvudally point then I sent this value using mqtt with below codes.
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
message = str(pose_3dqt[0][0])+"/n"+str(pose_3dqt[0][1])+"/n"+str(pose_3dqt[0][2])+"/n" message = message + str(pose_3dqt[1][0])+"/n"+str(pose_3dqt[1][1])+"/n"+str(pose_3dqt[1][2])+"/n" message = message + str(pose_3dqt[2][0])+"/n"+str(pose_3dqt[2][1])+"/n"+str(pose_3dqt[2][2])+"/n" message = message + str(pose_3dqt[3][0])+"/n"+str(pose_3dqt[3][1])+"/n"+str(pose_3dqt[3][2])+"/n" message = message + str(pose_3dqt[4][0])+"/n"+str(pose_3dqt[4][1])+"/n"+str(pose_3dqt[4][2])+"/n" message = message + str(pose_3dqt[5][0])+"/n"+str(pose_3dqt[5][1])+"/n"+str(pose_3dqt[5][2])+"/n" message = message + str(pose_3dqt[6][0])+"/n"+str(pose_3dqt[6][1])+"/n"+str(pose_3dqt[6][2])+"/n" message = message + str(pose_3dqt[7][0])+"/n"+str(pose_3dqt[7][1])+"/n"+str(pose_3dqt[7][2])+"/n" message = message + str(pose_3dqt[8][0])+"/n"+str(pose_3dqt[8][1])+"/n"+str(pose_3dqt[8][2])+"/n" message = message + str(pose_3dqt[9][0])+"/n"+str(pose_3dqt[9][1])+"/n"+str(pose_3dqt[9][2])+"/n" message = message + str(pose_3dqt[10][0])+"/n"+str(pose_3dqt[10][1])+"/n"+str(pose_3dqt[10][2])+"/n" message = message + str(pose_3dqt[11][0])+"/n"+str(pose_3dqt[11][1])+"/n"+str(pose_3dqt[11][2])+"/n" message = message + str(pose_3dqt[12][0])+"/n"+str(pose_3dqt[12][1])+"/n"+str(pose_3dqt[12][2])+"/n" message = message + str(pose_3dqt[13][0])+"/n"+str(pose_3dqt[13][1])+"/n"+str(pose_3dqt[13][2])+"/n" message = message + str(pose_3dqt[14][0])+"/n"+str(pose_3dqt[14][1])+"/n"+str(pose_3dqt[14][2])+"/n" message = message + str(pose_3dqt[15][0])+"/n"+str(pose_3dqt[15][1])+"/n"+str(pose_3dqt[15][2])+"/n" message = message + str(pose_3dqt[16][0])+"/n"+str(pose_3dqt[16][1])+"/n"+str(pose_3dqt[16][2])+"/n" client.publish("hello/world",message)#publish |
In the Unity side, I receive these value and separate them. After that, I update the position of spheres ;
|
1 |
Spheres[0].transform.position = new Vector3 (x0,y0,z0); |
x0,y0,z0 is the first keypoint ‘s positions address.
Full Code of Unity side is the bellow of this page.
When the start code for image output like these.
In next step, I will send real-time values and real-time 3D pose on Unity and communicate these two with the Internet.
FULL CODE C#
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
using UnityEngine; using System.Collections; using System.Net; using uPLibrary.Networking.M2Mqtt; using uPLibrary.Networking.M2Mqtt.Messages; using uPLibrary.Networking.M2Mqtt.Utility; using uPLibrary.Networking.M2Mqtt.Exceptions; using System; public class MovingParticles : MonoBehaviour { private MqttClient client; public GameObject[] Spheres = new GameObject[17]; private string value; float x0=0, x1 = 0, x2 = 0, x3 = 0, x4 = 0, x5 = 0, x6 = 0, x7 = 0, x8 = 0, x9 = 0, x10 = 0, x11 = 0, x12 = 0, x13 = 0, x14 = 0, x15 = 0, x16 = 0; float y0 = 0, y1 = 0, y2 = 0, y3 = 0, y4 = 0, y5 = 0, y6 = 0, y7 = 0, y8 = 0, y9 = 0, y10 = 0, y11 = 0, y12 = 0, y13 = 0, y14 = 0, y15 = 0, y16 = 0; float z0 = 0, z1 = 0, z2 = 0, z3 = 0, z4 = 0, z5 = 0, z6 = 0, z7 = 0, z8 = 0, z9 = 0, z10 = 0, z11 = 0, z12 = 0, z13 = 0, z14 = 0, z15 = 0, z16 = 0; Vector3[] position; // Use this for initialization void Start() { // create client instance client = new MqttClient(IPAddress.Parse("172.16.145.251"), 1883, false, null); // register to message received client.MqttMsgPublishReceived += client_MqttMsgPublishReceived; string clientId = Guid.NewGuid().ToString(); client.Connect(clientId); // subscribe to the topic "/home/temperature" with QoS 2 client.Subscribe(new string[] { "hello/world" }, new byte[] { MqttMsgBase.QOS_LEVEL_EXACTLY_ONCE }); } void client_MqttMsgPublishReceived(object sender, MqttMsgPublishEventArgs e) { value = System.Text.Encoding.UTF8.GetString(e.Message); Debug.Log(value); string[] lines = value.Split(new[] { "/n" }, StringSplitOptions.None); x0 = float.Parse(lines[0]) / 100; y0 = float.Parse(lines[1]) / 100; z0 = float.Parse(lines[2]) / 100; x1 = float.Parse(lines[3]) / 100; y1 = float.Parse(lines[4]) / 100; z1 = float.Parse(lines[5]) / 100; x2 = float.Parse(lines[6]) / 100; y2 = float.Parse(lines[7]) / 100; z2 = float.Parse(lines[8]) / 100; x3 = float.Parse(lines[9]) / 100; y3 = float.Parse(lines[10]) / 100; z3 = float.Parse(lines[11]) / 100; x4 = float.Parse(lines[12]) / 100; y4 = float.Parse(lines[13]) / 100; z4 = float.Parse(lines[14]) / 100; x5 = float.Parse(lines[15]) / 100; y5 = float.Parse(lines[16]) / 100; z5 = float.Parse(lines[17]) / 100; x6 = float.Parse(lines[18]) / 100; y6 = float.Parse(lines[19]) / 100; z6 = float.Parse(lines[20]) / 100; x7 = float.Parse(lines[21]) / 100; y7 = float.Parse(lines[22]) / 100; z7 = float.Parse(lines[23]) / 100; x8 = float.Parse(lines[24]) / 100; y8 = float.Parse(lines[25]) / 100; z8 = float.Parse(lines[26]) / 100; x9 = float.Parse(lines[27]) / 100; y9 = float.Parse(lines[28]) / 100; z9 = float.Parse(lines[29]) / 100; x10 = float.Parse(lines[30]) / 100; y10 = float.Parse(lines[31]) / 100; z10 = float.Parse(lines[32]) / 100; x11 = float.Parse(lines[33]) / 100; y11 = float.Parse(lines[34]) / 100; z11 = float.Parse(lines[35]) / 100; x12 = float.Parse(lines[36]) / 100; y12 = float.Parse(lines[37]) / 100; z12 = float.Parse(lines[38]) / 100; x13 = float.Parse(lines[39]) / 100; y13 = float.Parse(lines[40]) / 100; z13 = float.Parse(lines[41]) / 100; x14 = float.Parse(lines[42]) / 100; y14 = float.Parse(lines[43]) / 100; z14 = float.Parse(lines[44]) / 100; x15 = float.Parse(lines[45]) / 100; y15 = float.Parse(lines[46]) / 100; z15 = float.Parse(lines[47]) / 100; x16 = float.Parse(lines[48]) / 100; y16 = float.Parse(lines[49]) / 100; z16 = float.Parse(lines[50]) / 100; } // Update is called once per frame void Update() { Spheres[0].transform.position = new Vector3 (x0,y0,z0); Spheres[1].transform.position = new Vector3(x1, y1, z1); Spheres[2].transform.position = new Vector3(x2, y2, z2); Spheres[3].transform.position = new Vector3(x3, y3, z3); Spheres[4].transform.position = new Vector3(x4, y4, z4); Spheres[5].transform.position = new Vector3(x5, y5, z5); Spheres[6].transform.position = new Vector3(x6, y6, z6); Spheres[7].transform.position = new Vector3(x7, y7, z7); Spheres[8].transform.position = new Vector3(x8, y8, z8); Spheres[9].transform.position = new Vector3(x9, y9, z9); Spheres[10].transform.position = new Vector3(x10, y10, z10); Spheres[11].transform.position = new Vector3(x11, y11, z11); Spheres[12].transform.position = new Vector3(x12, y12, z12); Spheres[13].transform.position = new Vector3(x13, y13, z13); Spheres[14].transform.position = new Vector3(x14, y14, z14); Spheres[15].transform.position = new Vector3(x15, y15, z15); Spheres[16].transform.position = new Vector3(x16, y16, z16); } } |