From bf98b8256a3f171a7934e928cbcc998e068394b8 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Sat, 31 Dec 2022 20:31:09 -0300 Subject: [PATCH 1/4] Adding description as a first-class attribute for features/fields Signed-off-by: Amom Mendes --- protos/feast/core/Feature.proto | 4 ++++ protos/feast/types/Field.proto | 6 +++++ sdk/python/feast/feature.py | 11 +++++++++ sdk/python/feast/field.py | 8 ++++--- .../local/feature_repo/example_repo.py | 2 +- sdk/python/tests/unit/test_feature.py | 23 +++++++++++++++++++ ui/src/pages/features/FeatureOverviewTab.tsx | 5 ++++ 7 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 sdk/python/tests/unit/test_feature.py diff --git a/protos/feast/core/Feature.proto b/protos/feast/core/Feature.proto index f6826bef810..882de47eb9c 100644 --- a/protos/feast/core/Feature.proto +++ b/protos/feast/core/Feature.proto @@ -33,4 +33,8 @@ message FeatureSpecV2 { // Tags for user defined metadata on a feature map tags = 3; + + // Description of the feature. + + string description = 4; } diff --git a/protos/feast/types/Field.proto b/protos/feast/types/Field.proto index 8349263cc60..e32514b4004 100644 --- a/protos/feast/types/Field.proto +++ b/protos/feast/types/Field.proto @@ -27,4 +27,10 @@ option go_package = "github.com/feast-dev/feast/go/protos/feast/types"; message Field { string name = 1; feast.types.ValueType.Enum value = 2; + + // Tags for user defined metadata on a field + map tags = 3; + + // Description of the field. + string description = 4; } diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index 6b5acd9fc64..b9197065449 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -33,6 +33,7 @@ def __init__( self, name: str, dtype: ValueType, + description: str = "", labels: Optional[Dict[str, str]] = None, ): """Creates a Feature object.""" @@ -42,6 +43,7 @@ def __init__( if dtype is ValueType.UNKNOWN: raise ValueError(f"dtype cannot be {dtype}") self._dtype = dtype + self._description = description if labels is None: self._labels = dict() else: @@ -77,6 +79,13 @@ def dtype(self) -> ValueType: """ return self._dtype + @property + def description(self) -> str: + """ + Gets the description of the feature + """ + return self._description + @property def labels(self) -> Dict[str, str]: """ @@ -96,6 +105,7 @@ def to_proto(self) -> FeatureSpecProto: return FeatureSpecProto( name=self.name, value_type=value_type, + description=self.description, tags=self.labels, ) @@ -111,6 +121,7 @@ def from_proto(cls, feature_proto: FeatureSpecProto): feature = cls( name=feature_proto.name, dtype=ValueType(feature_proto.value_type), + description=feature_proto.description, labels=dict(feature_proto.tags), ) diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index a3dc3732da7..fd0121ea47b 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -30,7 +30,7 @@ class Field: Attributes: name: The name of the field. dtype: The type of the field, such as string or float. - tags: User-defined metadata in dictionary form. + tags (optional): User-defined metadata in dictionary form. """ name: str @@ -42,6 +42,7 @@ def __init__( *, name: str, dtype: FeastType, + description: str = "", tags: Optional[Dict[str, str]] = None, ): """ @@ -54,6 +55,7 @@ def __init__( """ self.name = name self.dtype = dtype + self.description = description self.tags = tags or {} def __eq__(self, other): @@ -83,7 +85,7 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" value_type = self.dtype.to_value_type() - return FieldProto(name=self.name, value_type=value_type.value, tags=self.tags) + return FieldProto(name=self.name, value_type=value_type.value, description=self.description, tags=self.tags) @classmethod def from_proto(cls, field_proto: FieldProto): @@ -109,5 +111,5 @@ def from_feature(cls, feature: Feature): feature: Feature object to convert. """ return cls( - name=feature.name, dtype=from_value_type(feature.dtype), tags=feature.labels + name=feature.name, dtype=from_value_type(feature.dtype), description=feature.description, tags=feature.labels ) diff --git a/sdk/python/feast/templates/local/feature_repo/example_repo.py b/sdk/python/feast/templates/local/feature_repo/example_repo.py index 131f1bcaa61..5aed3371b14 100644 --- a/sdk/python/feast/templates/local/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/local/feature_repo/example_repo.py @@ -45,7 +45,7 @@ schema=[ Field(name="conv_rate", dtype=Float32), Field(name="acc_rate", dtype=Float32), - Field(name="avg_daily_trips", dtype=Int64), + Field(name="avg_daily_trips", dtype=Int64, description="Average daily trips"), ], online=True, source=driver_stats_source, diff --git a/sdk/python/tests/unit/test_feature.py b/sdk/python/tests/unit/test_feature.py new file mode 100644 index 00000000000..6efdea77473 --- /dev/null +++ b/sdk/python/tests/unit/test_feature.py @@ -0,0 +1,23 @@ + + +from feast.field import Feature, Field +from feast.types import Float32 +from feast.value_type import ValueType + +def test_feature_serialization_with_description(): + expected_description = "Average daily trips" + feature = Feature(name="avg_daily_trips", dtype=ValueType.FLOAT, description=expected_description) + serialized_feature = feature.to_proto() + + assert serialized_feature.description == expected_description + +def test_field_serialization_with_description(): + expected_description = "Average daily trips" + field = Field(name="avg_daily_trips", dtype=Float32, description=expected_description) + feature = Feature(name="avg_daily_trips", dtype=ValueType.FLOAT, description=expected_description) + + serialized_field = field.to_proto() + field_from_feature = Field.from_feature(feature) + + assert serialized_field.description == expected_description + assert field_from_feature.description == expected_description \ No newline at end of file diff --git a/ui/src/pages/features/FeatureOverviewTab.tsx b/ui/src/pages/features/FeatureOverviewTab.tsx index 92adfe1501d..e339c30fc97 100644 --- a/ui/src/pages/features/FeatureOverviewTab.tsx +++ b/ui/src/pages/features/FeatureOverviewTab.tsx @@ -55,6 +55,11 @@ const FeatureOverviewTab = () => { {feast.types.ValueType.Enum[featureData?.valueType!]} + Description + + {featureData?.description} + + FeatureView Date: Sat, 31 Dec 2022 21:11:40 -0300 Subject: [PATCH 2/4] Formatting Signed-off-by: Amom Mendes --- sdk/python/feast/field.py | 12 ++++++++++-- sdk/python/tests/unit/test_feature.py | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index fd0121ea47b..d3bdf563923 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -85,7 +85,12 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" value_type = self.dtype.to_value_type() - return FieldProto(name=self.name, value_type=value_type.value, description=self.description, tags=self.tags) + return FieldProto( + name=self.name, + value_type=value_type.value, + description=self.description, + tags=self.tags, + ) @classmethod def from_proto(cls, field_proto: FieldProto): @@ -111,5 +116,8 @@ def from_feature(cls, feature: Feature): feature: Feature object to convert. """ return cls( - name=feature.name, dtype=from_value_type(feature.dtype), description=feature.description, tags=feature.labels + name=feature.name, + dtype=from_value_type(feature.dtype), + description=feature.description, + tags=feature.labels, ) diff --git a/sdk/python/tests/unit/test_feature.py b/sdk/python/tests/unit/test_feature.py index 6efdea77473..a8cfeef3dab 100644 --- a/sdk/python/tests/unit/test_feature.py +++ b/sdk/python/tests/unit/test_feature.py @@ -1,23 +1,29 @@ - - from feast.field import Feature, Field from feast.types import Float32 from feast.value_type import ValueType + def test_feature_serialization_with_description(): expected_description = "Average daily trips" - feature = Feature(name="avg_daily_trips", dtype=ValueType.FLOAT, description=expected_description) + feature = Feature( + name="avg_daily_trips", dtype=ValueType.FLOAT, description=expected_description + ) serialized_feature = feature.to_proto() assert serialized_feature.description == expected_description + def test_field_serialization_with_description(): expected_description = "Average daily trips" - field = Field(name="avg_daily_trips", dtype=Float32, description=expected_description) - feature = Feature(name="avg_daily_trips", dtype=ValueType.FLOAT, description=expected_description) + field = Field( + name="avg_daily_trips", dtype=Float32, description=expected_description + ) + feature = Feature( + name="avg_daily_trips", dtype=ValueType.FLOAT, description=expected_description + ) serialized_field = field.to_proto() field_from_feature = Field.from_feature(feature) assert serialized_field.description == expected_description - assert field_from_feature.description == expected_description \ No newline at end of file + assert field_from_feature.description == expected_description From ba4df5cafa904e4a6fdacb2f1db064d4e8a9d67c Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Fri, 6 Jan 2023 22:38:56 -0300 Subject: [PATCH 3/4] Add list validation references Signed-off-by: Amom Mendes --- .../feast/infra/registry/proto_registry_utils.py | 4 ++++ sdk/python/feast/infra/registry/registry.py | 10 ++++++++++ sdk/python/feast/infra/registry/sql.py | 16 ++++++++++++++++ .../unit/local_feast_tests/test_e2e_local.py | 2 +- 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index e26b1e10e79..bff5fc7a894 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -115,6 +115,10 @@ def get_validation_reference( return ValidationReference.from_proto(validation_reference) raise ValidationReferenceNotFound(name, project=project) +def list_validation_references( + registry_proto: RegistryProto +): + return registry_proto.validation_references def list_feature_services( registry_proto: RegistryProto, project: str, allow_cache: bool = False diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index 4870c1d45d4..fe44010f460 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -740,6 +740,16 @@ def get_validation_reference( registry_proto, name, project ) + def list_validation_references( + self, project: str, allow_cache: bool = False + ) -> List[ValidationReference]: + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) + return proto_registry_utils.list_validation_references( + registry_proto + ) + def delete_validation_reference(self, name: str, project: str, commit: bool = True): registry_proto = self._prepare_registry_for_changes(project) for idx, existing_validation_reference in enumerate( diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index f0782515443..507020d3e54 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -408,6 +408,22 @@ def get_validation_reference( not_found_exception=ValidationReferenceNotFound, ) + def list_validation_references( + self, project: str, allow_cache: bool = False + ) -> List[ValidationReference]: + if allow_cache: + self._refresh_cached_registry_if_necessary() + return proto_registry_utils.list_validation_references( + self.cached_registry_proto + ) + return self._list_objects( + table=validation_references, + project=project, + proto_class=ValidationReferenceProto, + python_class=ValidationReference, + proto_field_name="validation_reference_proto" + ) + def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: if allow_cache: self._refresh_cached_registry_if_necessary() diff --git a/sdk/python/tests/unit/local_feast_tests/test_e2e_local.py b/sdk/python/tests/unit/local_feast_tests/test_e2e_local.py index 1ead69f52a0..fa272c4847f 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_e2e_local.py +++ b/sdk/python/tests/unit/local_feast_tests/test_e2e_local.py @@ -21,7 +21,7 @@ def test_e2e_local() -> None: """ Tests the end-to-end workflow of apply, materialize, and online retrieval. - This test runs against several different types of repos: + This test runs against several types of repos: 1. A repo with a normal FV and an entity-less FV. 2. A repo using the SDK from version 0.19.0. 3. A repo with a FV with a ttl of 0. From a4df6774334cadb945ff565fef979b12b519bb33 Mon Sep 17 00:00:00 2001 From: Amom Mendes Date: Fri, 6 Jan 2023 22:54:32 -0300 Subject: [PATCH 4/4] Format Signed-off-by: Amom Mendes --- sdk/python/feast/infra/registry/proto_registry_utils.py | 6 +++--- sdk/python/feast/infra/registry/registry.py | 6 ++---- sdk/python/feast/infra/registry/sql.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index bff5fc7a894..4dbc95d2a5a 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -115,11 +115,11 @@ def get_validation_reference( return ValidationReference.from_proto(validation_reference) raise ValidationReferenceNotFound(name, project=project) -def list_validation_references( - registry_proto: RegistryProto -): + +def list_validation_references(registry_proto: RegistryProto): return registry_proto.validation_references + def list_feature_services( registry_proto: RegistryProto, project: str, allow_cache: bool = False ) -> List[FeatureService]: diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index fe44010f460..e2e98460595 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -741,14 +741,12 @@ def get_validation_reference( ) def list_validation_references( - self, project: str, allow_cache: bool = False + self, project: str, allow_cache: bool = False ) -> List[ValidationReference]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_validation_references( - registry_proto - ) + return proto_registry_utils.list_validation_references(registry_proto) def delete_validation_reference(self, name: str, project: str, commit: bool = True): registry_proto = self._prepare_registry_for_changes(project) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 507020d3e54..fd8b47cd81a 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -421,7 +421,7 @@ def list_validation_references( project=project, proto_class=ValidationReferenceProto, python_class=ValidationReference, - proto_field_name="validation_reference_proto" + proto_field_name="validation_reference_proto", ) def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: