Merge pull request #115 from threedworld-mit/screen_position

Screen position

Documentation/Changelog.md

@@ -8,12 +8,26 @@ To upgrade from TDW v1.6 to v1.7, read [this guide](Documentation/v1.6_to_v1.7).
 
 ### Command API
 
+#### New Commands
+
+| Command                 | Description                                                  |
+| ----------------------- | ------------------------------------------------------------ |
+| `send_screen_positions` | Given a list of worldspace positions, return the screenspace positions according to each of the avatar's camera. |
+
 #### Modified Commands
 
 | Command               | Modification                                                 |
 | --------------------- | ------------------------------------------------------------ |
 | `add_position_marker` | Fixed: The `a` value of `color` doesn't adjust the transparency.<br>Added: optional parameter `shape`. |
 
+### Output Data
+
+#### New Output Data
+
+| Output Data      | Description                                       |
+| ---------------- | ------------------------------------------------- |
+| `ScreenPosition` | A worldspace position in screenspace coordinates. |
+
 ## v1.7.10
 
 ### Command API

Documentation/api/command_api.md

@@ -430,6 +430,7 @@
 | [`send_id_pass_segmentation_colors`](#send_id_pass_segmentation_colors) | Send all unique colors in an _id pass.  |
 | [`send_images`](#send_images) | Send images and metadata.  |
 | [`send_image_sensors`](#send_image_sensors) | Send data about each of the avatar's ImageSensors.  |
+| [`send_screen_positions`](#send_screen_positions) | Given a list of worldspace positions, return the screenspace positions according to each of the avatar's camera.  |
 
 **Send Overlap Command**
 
@@ -5174,6 +5175,41 @@ Options for when to send data.
 | `"always"` | Send the data every frame. |
 | `"never"` | Never send the data. |
 
+***
+
+## **`send_screen_positions`**
+
+Given a list of worldspace positions, return the screenspace positions according to each of the avatar's camera. 
+
+- <font style="color:green">**Sends data**: This command instructs the build to send output data.</font>
+
+    - <font style="color:green">**Type:** [`ScreenPosition`](output_data.md#ScreenPosition)</font>
+
+```python
+{"$type": "send_screen_positions", "position_ids": [0, 1, 2], "positions": [{"x": 1.1, "y": 0.0, "z": 0}, {"x": 2, "y": 0, "z": -1}]}
+```
+
+```python
+{"$type": "send_screen_positions", "position_ids": [0, 1, 2], "positions": [{"x": 1.1, "y": 0.0, "z": 0}, {"x": 2, "y": 0, "z": -1}], "ids": [], "frequency": "once"}
+```
+
+| Parameter | Type | Description | Default |
+| --- | --- | --- | --- |
+| `"position_ids"` | int [] | The unique IDs of each screen position output data. Use this to map the output data to the original worldspace position. | |
+| `"positions"` | Vector3 [] | The worldspace positions. | |
+| `"ids"` | string[] | The IDs of the avatars. If this list is undefined or empty, the build will return data for all avatars. | [] |
+| `"frequency"` | Frequency | The frequency at which data is sent. | "once" |
+
+#### Frequency
+
+Options for when to send data.
+
+| Value | Description |
+| --- | --- |
+| `"once"` | Send the data for this frame only. |
+| `"always"` | Send the data every frame. |
+| `"never"` | Never send the data. |
+
 # SendOverlapCommand
 
 These commands create an overlap shape and then check which objects are within that shape.

Documentation/api/output_data.md

@@ -77,6 +77,7 @@ Objects in arrays can't be directly accessed (this is due to how the backend cod
 | [Overlap](#Overlap) | The IDs of every object that a shape overlaps. | `over` |
 | [Raycast](#Raycast) | A ray cast from an origin to a destination and what, if anything, it hit. | `rayc` |
 | [Rigidbodies](#Rigidbodies) | Rigibody data (velocity, mass, etc.) for objects in the scene. | `rigi` |
+| [ScreenPosition](#ScreenPosition) | A position on the screen converted from a worldspace position. | `scre` |
 | [SegmentationColors](#SegmentationColors) | Color segmentation data for objects in the scene. | `segm` |
 | [Substructure](#Substructure) | The substructure of a model. This should be used mainly for backend debugging. | `subs` |
 | [Transforms](#Transforms) | Data about the Transform component of objects (position and rotation). | `tran` |
@@ -532,6 +533,22 @@ Rigibody data (velocity, mass, etc.) for objects in the scene.
 | `get_mass(index)` | The mass. | `float` |
 | `get_sleeping(index)` | True if the rigidbody is sleeping. | `bool` |
 
+## ScreenPosition
+
+`s = ScreenPosition(byte_array)`
+
+**Identifier:** `scre`
+
+A position on the screen converted from a worldspace position.
+
+| Function | Description | Return type |
+| --- | --- | --- |
+| `get_avatar_id()` | The ID of the avatar that is rendered the screen. | `str` |
+| `get_sensor_name()` | The name of the sensor that rendered the screen. | `str` |
+| `get_id()` | An identifier for the screen position to help you map it back to the original world position. | `int` |
+| `get_screen()` | The position in screenspace coordinates. From the [Unity documentation:](https: | `Tuple[float, float, float]` |
+| `get_world()` | The position in worldspace coordinates. | `Tuple[float, float, float]` |
+
 ## SegmentationColors
 
 `s = SegmentationColors(byte_array)`

Python/tdw/FBOutput/ScreenPosition.py

@@ -0,0 +1,70 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: FBOutput
+
+import tdw.flatbuffers
+
+class ScreenPosition(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsScreenPosition(cls, buf, offset):
+        n = tdw.flatbuffers.encode.Get(tdw.flatbuffers.packer.uoffset, buf, offset)
+        x = ScreenPosition()
+        x.Init(buf, n + offset)
+        return x
+
+    # ScreenPosition
+    def Init(self, buf, pos):
+        self._tab = tdw.flatbuffers.table.Table(buf, pos)
+
+    # ScreenPosition
+    def AvatarId(self):
+        o = tdw.flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ScreenPosition
+    def SensorName(self):
+        o = tdw.flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # ScreenPosition
+    def Id(self):
+        o = tdw.flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.Get(tdw.flatbuffers.number_types.Int32Flags, o + self._tab.Pos)
+        return 0
+
+    # ScreenPosition
+    def World(self):
+        o = tdw.flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            x = o + self._tab.Pos
+            from .Vector3 import Vector3
+            obj = Vector3()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # ScreenPosition
+    def Screen(self):
+        o = tdw.flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            x = o + self._tab.Pos
+            from .Vector3 import Vector3
+            obj = Vector3()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+def ScreenPositionStart(builder): builder.StartObject(5)
+def ScreenPositionAddAvatarId(builder, avatarId): builder.PrependUOffsetTRelativeSlot(0, tdw.flatbuffers.number_types.UOffsetTFlags.py_type(avatarId), 0)
+def ScreenPositionAddSensorName(builder, sensorName): builder.PrependUOffsetTRelativeSlot(1, tdw.flatbuffers.number_types.UOffsetTFlags.py_type(sensorName), 0)
+def ScreenPositionAddId(builder, id): builder.PrependInt32Slot(2, id, 0)
+def ScreenPositionAddWorld(builder, world): builder.PrependStructSlot(3, tdw.flatbuffers.number_types.UOffsetTFlags.py_type(world), 0)
+def ScreenPositionAddScreen(builder, screen): builder.PrependStructSlot(4, tdw.flatbuffers.number_types.UOffsetTFlags.py_type(screen), 0)
+def ScreenPositionEnd(builder): return builder.EndObject()

Python/tdw/output_data.py

@@ -32,6 +32,7 @@
 from tdw.FBOutput import Overlap as Over
 from tdw.FBOutput import NavMeshPath as Path
 from tdw.FBOutput import Keyboard as Key
+from tdw.FBOutput import ScreenPosition as Screen
 import numpy as np
 from typing import Tuple, Optional
 
@@ -820,3 +821,23 @@ def get_num_released(self) -> int:
 
     def get_released(self, index: int) -> str:
         return self.data.Released(index).decode('utf-8')
+
+
+class ScreenPosition(OutputData):
+    def get_data(self) -> Screen.ScreenPosition:
+        return Screen.ScreenPosition.GetRootAsScreenPosition(self.bytes, 0)
+
+    def get_avatar_id(self) -> str:
+        return self.data.AvatarId().decode('utf-8')
+
+    def get_sensor_name(self) -> str:
+        return self.data.SensorName().decode('utf-8')
+
+    def get_id(self) -> int:
+        return self.data.Id()
+
+    def get_screen(self) -> Tuple[float, float, float]:
+        return OutputData._get_xyz(self.data.Screen())
+
+    def get_world(self) -> Tuple[float, float, float]:
+        return OutputData._get_xyz(self.data.World())