docarray · samsja · Dec 14, 2022 · Dec 9, 2022 · Dec 9, 2022 · Dec 9, 2022
diff --git a/docarray/__init__.py b/docarray/__init__.py
@@ -2,6 +2,6 @@
 
 from docarray.array import DocumentArray
 from docarray.document.document import BaseDocument as Document
-from docarray.predefined_document import Image, Text
+from docarray.predefined_document import Image, Mesh3D, PointCloud3D, Text
 
-__all__ = ['Document', 'DocumentArray', 'Image', 'Text']
+__all__ = ['Document', 'DocumentArray', 'Image', 'Text', 'Mesh3D', 'PointCloud3D']
diff --git a/docarray/document/mixins/proto.py b/docarray/document/mixins/proto.py
@@ -29,7 +29,9 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocumentProto') -> T:
             AnyUrl,
             Embedding,
             ImageUrl,
+            Mesh3DUrl,
             NdArray,
+            PointCloud3DUrl,
             TextUrl,
         )
 
@@ -48,6 +50,8 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocumentProto') -> T:
                 any_url=AnyUrl,
                 text_url=TextUrl,
                 image_url=ImageUrl,
+                mesh_url=Mesh3DUrl,
+                point_cloud_url=PointCloud3DUrl,
                 id=ID,
             )
 

diff --git a/docarray/predefined_document/__init__.py b/docarray/predefined_document/__init__.py
@@ -1,4 +1,6 @@
 from docarray.predefined_document.image import Image
+from docarray.predefined_document.mesh import Mesh3D
+from docarray.predefined_document.point_cloud import PointCloud3D
 from docarray.predefined_document.text import Text
 
-__all__ = ['Text', 'Image']
+__all__ = ['Text', 'Image', 'Mesh3D', 'PointCloud3D']
diff --git a/docarray/predefined_document/mesh.py b/docarray/predefined_document/mesh.py
@@ -0,0 +1,78 @@
+from typing import Optional
+
+from docarray.document import BaseDocument
+from docarray.typing import Embedding, Mesh3DUrl, Tensor
+
+
+class Mesh3D(BaseDocument):
+    """
+    Document for handling meshes for 3D data representation.
+
+    A mesh is a representation for 3D data and contains vertices and faces information.
+    Vertices are points in a 3D space, represented as a tensor of shape (n_points, 3).
+    Faces are triangular surfaces that can be defined by three points in 3D space,
+    corresponding to the three vertices of a triangle. Faces can be represented as a
+    tensor of shape (n_faces, 3). Each number in that tensor refers to an index of a
+    vertex in the tensor of vertices.
+
+    The Mesh3D Document can contain an Mesh3DUrl (`Mesh3D.url`), a Tensor of vertices
+    (`Mesh3D.vertices`), a Tensor of faces (`Mesh3D.faces`) and an Embedding
+    (`Mesh3D.embedding`).
+
+    EXAMPLE USAGE:
+
+    You can use this Document directly:
+
+    .. code-block:: python
+
+        from docarray import Mesh3D
+
+        # use it directly
+        mesh = Mesh3D(url='https://people.sc.fsu.edu/~jburkardt/data/obj/al.obj')
+        mesh.vertices, mesh.faces = mesh.url.load()
+        model = MyEmbeddingModel()
+        mesh.embedding = model(mesh.vertices)
+
+    You can extend this Document:
+
+    .. code-block:: python
+
+        from docarray import Mesh3D
+        from docarray.typing import Embedding
+        from typing import Optional
+
+        # extend it
+        class MyMesh3D(Mesh3D):
+            name: Optional[Text]
+
+
+        mesh = MyMesh3D(url='https://people.sc.fsu.edu/~jburkardt/data/obj/al.obj')
+        mesh.vertices, mesh.faces = mesh.url.load()
+        model = MyEmbeddingModel()
+        mesh.embedding = model(mesh.vertices)
+        mesh.name = 'my first mesh'
+
+
+    You can use this Document for composition:
+
+    .. code-block:: python
+
+        from docarray import Document, Mesh3D, Text
+
+        # compose it
+        class MultiModalDoc(Document):
+            mesh: Mesh3D
+            text: Text
+
+
+        mmdoc = MultiModalDoc(
+            mesh=Mesh3D(url='https://people.sc.fsu.edu/~jburkardt/data/obj/al.obj'),
+            text=Text(text='hello world, how are you doing?'),
+        )
+        mmdoc.mesh.vertices, mmdoc.mesh.faces = mmdoc.mesh.url.load()
+    """
+
+    url: Optional[Mesh3DUrl]
+    vertices: Optional[Tensor]
+    faces: Optional[Tensor]
+    embedding: Optional[Embedding]
diff --git a/docarray/predefined_document/point_cloud.py b/docarray/predefined_document/point_cloud.py
@@ -0,0 +1,76 @@
+from typing import Optional
+
+from docarray.document import BaseDocument
+from docarray.typing import Embedding, PointCloud3DUrl, Tensor
+
+
+class PointCloud3D(BaseDocument):
+    """
+    Document for handling point clouds for 3D data representation.
+
+    Point cloud is a representation of a 3D mesh. It is made by repeatedly and uniformly
+    sampling points within the surface of the 3D body. Compared to the mesh
+    representation, the point cloud is a fixed size ndarray (shape=(n_samples, 3)) and
+    hence easier for deep learning algorithms to handle.
+
+    A PointCloud3D Document can contain an PointCloud3DUrl (`PointCloud3D.url`), a
+    Tensor (`PointCloud3D.tensor`), and an Embedding (`PointCloud3D.embedding`).
+
+    EXAMPLE USAGE:
+
+    You can use this Document directly:
+
+    .. code-block:: python
+
+        from docarray import PointCloud3D
+
+        # use it directly
+        pc = PointCloud3D(url='https://people.sc.fsu.edu/~jburkardt/data/obj/al.obj')
+        pc.tensor = pc.url.load(samples=100)
+        model = MyEmbeddingModel()
+        pc.embedding = model(pc.tensor)
+
+    You can extend this Document:
+
+    .. code-block:: python
+
+        from docarray import PointCloud3D
+        from docarray.typing import Embedding
+        from typing import Optional
+
+        # extend it
+        class MyPointCloud3D(PointCloud3D):
+            second_embedding: Optional[Embedding]
+
+
+        pc = MyPointCloud3D(url='https://people.sc.fsu.edu/~jburkardt/data/obj/al.obj')
+        pc.tensor = pc.url.load(samples=100)
+        model = MyEmbeddingModel()
+        pc.embedding = model(pc.tensor)
+        pc.second_embedding = model(pc.tensor)
+
+
+    You can use this Document for composition:
+
+    .. code-block:: python
+
+        from docarray import Document, PointCloud3D, Text
+
+        # compose it
+        class MultiModalDoc(Document):
+            point_cloud: PointCloud3D
+            text: Text
+
+
+        mmdoc = MultiModalDoc(
+            point_cloud=PointCloud3D(
+                url='https://people.sc.fsu.edu/~jburkardt/data/obj/al.obj'
+            ),
+            text=Text(text='hello world, how are you doing?'),
+        )
+        mmdoc.point_cloud.tensor = mmdoc.point_cloud.url.load(samples=100)
+    """
+
+    url: Optional[PointCloud3DUrl]
+    tensor: Optional[Tensor]
+    embedding: Optional[Embedding]
diff --git a/docarray/proto/docarray.proto b/docarray/proto/docarray.proto
@@ -59,6 +59,11 @@ message NodeProto {
 
     NdArrayProto torch_tensor = 11;
 
+    string mesh_url = 12;
+
+    string point_cloud_url = 13;
+
+
   }
 
 }

diff --git a/docarray/proto/pb2/docarray_pb2.py b/docarray/proto/pb2/docarray_pb2.py
diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py
@@ -1,13 +1,15 @@
 from docarray.typing.id import ID
 from docarray.typing.tensor import NdArray, Tensor
 from docarray.typing.tensor.embedding import Embedding
-from docarray.typing.url import AnyUrl, ImageUrl, TextUrl
+from docarray.typing.url import AnyUrl, ImageUrl, Mesh3DUrl, PointCloud3DUrl, TextUrl
 
 __all__ = [
     'NdArray',
     'Embedding',
     'ImageUrl',
     'TextUrl',
+    'Mesh3DUrl',
+    'PointCloud3DUrl',
     'AnyUrl',
     'ID',
     'Tensor',

diff --git a/docarray/typing/abstract_type.py b/docarray/typing/abstract_type.py
@@ -1,11 +1,14 @@
 from abc import abstractmethod
-from typing import Any, Type, TypeVar
+from typing import TYPE_CHECKING, Any, Type, TypeVar
 
 from pydantic import BaseConfig
 from pydantic.fields import ModelField
 
 from docarray.document.base_node import BaseNode
 
+if TYPE_CHECKING:
+    from docarray.proto import NodeProto
+
 T = TypeVar('T')
 
 
@@ -28,3 +31,7 @@ def validate(
     @abstractmethod
     def from_protobuf(cls: Type[T], pb_msg: T) -> T:
         ...
+
+    @abstractmethod
+    def _to_node_protobuf(self: T) -> 'NodeProto':
+        ...
diff --git a/docarray/typing/url/__init__.py b/docarray/typing/url/__init__.py
@@ -1,5 +1,7 @@
 from docarray.typing.url.any_url import AnyUrl
 from docarray.typing.url.image_url import ImageUrl
 from docarray.typing.url.text_url import TextUrl
+from docarray.typing.url.url_3d.mesh_url import Mesh3DUrl
+from docarray.typing.url.url_3d.point_cloud_url import PointCloud3DUrl
 
-__all__ = ['ImageUrl', 'AnyUrl', 'TextUrl']
+__all__ = ['ImageUrl', 'AnyUrl', 'TextUrl', 'Mesh3DUrl', 'PointCloud3DUrl']
diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py
@@ -20,14 +20,14 @@
 
 
 class ImageUrl(AnyUrl):
-    """ "
+    """
     URL to a .png, .jpeg, or .jpg file.
-    Cane be remote (web) URL, or a local file path.
+    Can be remote (web) URL, or a local file path.
     """
 
     def _to_node_protobuf(self) -> 'NodeProto':
         """Convert Document into a NodeProto protobuf message. This function should
-        be called when the Document is nested into another Document that need to
+        be called when the Document is nested into another Document that needs to
         be converted into a protobuf
 
         :return: the nested item protobuf message

diff --git a/docarray/typing/url/url_3d/__init__.py b/docarray/typing/url/url_3d/__init__.py
@@ -0,0 +1,4 @@
+from docarray.typing.url.url_3d.mesh_url import Mesh3DUrl
+from docarray.typing.url.url_3d.point_cloud_url import PointCloud3DUrl
+
+__all__ = ['Mesh3DUrl', 'PointCloud3DUrl']
diff --git a/docarray/typing/url/url_3d/mesh_url.py b/docarray/typing/url/url_3d/mesh_url.py
@@ -0,0 +1,63 @@
+from typing import TYPE_CHECKING, Tuple, TypeVar
+
+import numpy as np
+
+from docarray.typing.url.url_3d.url_3d import Url3D
+
+if TYPE_CHECKING:
+    from docarray.proto import NodeProto
+
+T = TypeVar('T', bound='Mesh3DUrl')
+
+
+class Mesh3DUrl(Url3D):
+    """
+    URL to a .obj, .glb, or .ply file containing 3D mesh information.
+    Can be remote (web) URL, or a local file path.
+    """
+
+    def _to_node_protobuf(self: T) -> 'NodeProto':
+        """Convert Document into a NodeProto protobuf message. This function should
+        be called when the Document is nested into another Document that needs to
+        be converted into a protobuf
+
+        :return: the nested item protobuf message
+        """
+        from docarray.proto import NodeProto
+
+        return NodeProto(mesh_url=str(self))
+
+    def load(self: T) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Load the data from the url into a tuple of two numpy.ndarrays containing
+        vertices and faces information.
+
+        EXAMPLE USAGE
+
+        .. code-block:: python
+
+            from docarray import Document
+            import numpy as np
+
+            from docarray.typing import Mesh3DUrl
+
+
+            class MyDoc(Document):
+                mesh_url: Mesh3DUrl
+
+
+            doc = MyDoc(mesh_url="toydata/tetrahedron.obj")
+
+            vertices, faces = doc.mesh_url.load()
+            assert isinstance(vertices, np.ndarray)
+            assert isinstance(faces, np.ndarray)
+
+        :return: tuple of two np.ndarrays representing the mesh's vertices and faces
+        """
+
+        mesh = self._load_trimesh_instance(force='mesh')
+
+        vertices = mesh.vertices.view(np.ndarray)
+        faces = mesh.faces.view(np.ndarray)
+
+        return vertices, faces
-Original file line number
+Diff line change
@@ Expand Up / @@ -59,6 +59,11 @@ message NodeProto { @@
         NdArrayProto torch_tensor = 11;
+        string mesh_url = 12;
+        string point_cloud_url = 13;
       }
     }
@@ Expand Down @@