From da72c6d675ded9ff0cbbeb1ab5e25f2dccc6131a Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Mon, 26 Feb 2024 06:58:47 -0800 Subject: [PATCH] add serialization support (#34310) * add serialization support * updates * update changelog * updates * update * update * update * update * Update sdk/search/azure-search-documents/CHANGELOG.md Co-authored-by: Paul Van Eck * update * update * update --------- Co-authored-by: Paul Van Eck --- .../azure-search-documents/CHANGELOG.md | 5 + .../search/documents/indexes/models/_index.py | 102 ++++---- .../documents/indexes/models/_models.py | 233 +++++++++--------- .../tests/test_serialization.py | 79 ++++++ 4 files changed, 245 insertions(+), 174 deletions(-) create mode 100644 sdk/search/azure-search-documents/tests/test_serialization.py diff --git a/sdk/search/azure-search-documents/CHANGELOG.md b/sdk/search/azure-search-documents/CHANGELOG.md index 512211e184cd..f4603347136e 100644 --- a/sdk/search/azure-search-documents/CHANGELOG.md +++ b/sdk/search/azure-search-documents/CHANGELOG.md @@ -6,8 +6,12 @@ ### Breaking Changes +- `SearchIndexerSkillset`, `SearchField`, `SearchIndex`, `AnalyzeTextOptions`, `SearchResourceEncryptionKey`, `SynonymMap`, `SearchIndexerDataSourceConnection` are no longer subclasses of `_serialization.Model`. + ### Bugs Fixed +- Fixed the issue that `SearchIndexerSkillset`, `SearchField`, `SearchIndex`, `AnalyzeTextOptions`, `SearchResourceEncryptionKey`, `SynonymMap`, `SearchIndexerDataSourceConnection` could not be serialized. + ### Other Changes ## 11.6.0b1 (2024-01-31) @@ -76,6 +80,7 @@ - `alias` operations are not available in this stable release. - `AzureOpenAIEmbeddingSkill`, `AzureOpenAIParameters` and `AzureOpenAIVectorizer` are not available in 11.4.0. - Renamed `vector_search_profile` to `vector_search_profile_name` in `SearchField`. +- Renamed `SemanticSettings` to `SemanticSearch`. ### Other Changes diff --git a/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_index.py b/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_index.py index dd7c63205abc..28ede871d7c9 100644 --- a/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_index.py +++ b/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_index.py @@ -3,9 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import Any, Dict, Union, List, Optional +from typing import Any, Dict, Union, List, Optional, MutableMapping -from .._generated import _serialization from ._edm import Collection, ComplexType, String from .._generated.models import ( SearchField as _SearchField, @@ -24,7 +23,7 @@ __all__ = ("ComplexField", "SearchableField", "SimpleField") -class SearchField(_serialization.Model): +class SearchField: # pylint: disable=too-many-instance-attributes """Represents a field in an index definition, which describes the name, data type, and search behavior of a field. @@ -165,35 +164,7 @@ class SearchField(_serialization.Model): :vartype fields: list[~azure.search.documents.indexes.models.SearchField] """ - _validation = { - "name": {"required": True}, - "type": {"required": True}, - } - - _attribute_map = { - "name": {"key": "name", "type": "str"}, - "type": {"key": "type", "type": "str"}, - "key": {"key": "key", "type": "bool"}, - "hidden": {"key": "hidden", "type": "bool"}, - "searchable": {"key": "searchable", "type": "bool"}, - "filterable": {"key": "filterable", "type": "bool"}, - "sortable": {"key": "sortable", "type": "bool"}, - "facetable": {"key": "facetable", "type": "bool"}, - "analyzer_name": {"key": "analyzerName", "type": "str"}, - "search_analyzer_name": {"key": "searchAnalyzerName", "type": "str"}, - "index_analyzer_name": {"key": "indexAnalyzerName", "type": "str"}, - "normalizer_name": {"key": "normalizerName", "type": "str"}, - "synonym_map_names": {"key": "synonymMapNames", "type": "[str]"}, - "fields": {"key": "fields", "type": "[SearchField]"}, - "vector_search_dimensions": {"key": "vectorSearchDimensions", "type": "int"}, - "vector_search_profile_name": { - "key": "vectorSearchProfile", - "type": "str", - }, - } - def __init__(self, **kwargs): - super(SearchField, self).__init__(**kwargs) self.name = kwargs["name"] self.type = kwargs["type"] self.key = kwargs.get("key", None) @@ -263,6 +234,25 @@ def _from_generated(cls, search_field): vector_search_profile_name=search_field.vector_search_profile_name, ) + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_generated().serialize(keep_readonly=keep_readonly, **kwargs) + + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "SearchField": + """Parse a str using the RestAPI syntax and return a SearchField instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A SearchField instance + :raises: DeserializationError if something went wrong + """ + return cls._from_generated(_SearchField.deserialize(data, content_type=content_type)) # type: ignore + def SimpleField( *, @@ -515,7 +505,7 @@ def ComplexField( return SearchField(**result) -class SearchIndex(_serialization.Model): +class SearchIndex: # pylint: disable=too-many-instance-attributes """Represents a search index definition, which describes the fields and search behavior of an index. @@ -567,35 +557,7 @@ class SearchIndex(_serialization.Model): :vartype e_tag: str """ - _validation = { - "name": {"required": True}, - "fields": {"required": True}, - } - - _attribute_map = { - "name": {"key": "name", "type": "str"}, - "fields": {"key": "fields", "type": "[SearchField]"}, - "scoring_profiles": {"key": "scoringProfiles", "type": "[ScoringProfile]"}, - "default_scoring_profile": {"key": "defaultScoringProfile", "type": "str"}, - "cors_options": {"key": "corsOptions", "type": "CorsOptions"}, - "suggesters": {"key": "suggesters", "type": "[SearchSuggester]"}, - "analyzers": {"key": "analyzers", "type": "[LexicalAnalyzer]"}, - "tokenizers": {"key": "tokenizers", "type": "[LexicalTokenizer]"}, - "token_filters": {"key": "tokenFilters", "type": "[TokenFilter]"}, - "char_filters": {"key": "charFilters", "type": "[CharFilter]"}, - "normalizers": {"key": "normalizers", "type": "[LexicalNormalizer]"}, - "encryption_key": { - "key": "encryptionKey", - "type": "SearchResourceEncryptionKey", - }, - "similarity": {"key": "similarity", "type": "SimilarityAlgorithm"}, - "semantic_search": {"key": "semantic", "type": "SemanticSearch"}, - "vector_search": {"key": "vectorSearch", "type": "VectorSearch"}, - "e_tag": {"key": "@odata\\.etag", "type": "str"}, - } - def __init__(self, **kwargs): - super(SearchIndex, self).__init__(**kwargs) self.name = kwargs["name"] self.fields = kwargs["fields"] self.scoring_profiles = kwargs.get("scoring_profiles", None) @@ -692,6 +654,26 @@ def _from_generated(cls, search_index) -> "SearchIndex": vector_search=search_index.vector_search, ) + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_generated().serialize(keep_readonly=keep_readonly, **kwargs) + + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "SearchIndex": + """Parse a str using the RestAPI syntax and return a SearchIndex instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A SearchIndex instance + :rtype: SearchIndex + :raises: DeserializationError if something went wrong + """ + return cls._from_generated(_SearchIndex.deserialize(data, content_type=content_type)) + def pack_search_field(search_field: SearchField) -> _SearchField: if isinstance(search_field, dict): diff --git a/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_models.py b/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_models.py index ea6d094dc4a9..14d0ff763c5d 100644 --- a/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_models.py +++ b/sdk/search/azure-search-documents/azure/search/documents/indexes/models/_models.py @@ -4,10 +4,9 @@ # license information. # -------------------------------------------------------------------------- -from typing import Any, List, Optional +from typing import Any, List, Optional, MutableMapping from enum import Enum from azure.core import CaseInsensitiveEnumMeta -from .._generated import _serialization from .._generated.models import ( LexicalAnalyzer, LexicalTokenizer, @@ -29,11 +28,10 @@ SearchIndexerIndexProjections, ) - DELIMITER = "|" -class SearchIndexerSkillset(_serialization.Model): +class SearchIndexerSkillset: """A list of skills. All required parameters must be populated in order to send to Azure. @@ -65,34 +63,6 @@ class SearchIndexerSkillset(_serialization.Model): :vartype encryption_key: ~azure.search.documents.indexes.models.SearchResourceEncryptionKey """ - _validation = { - "name": {"required": True}, - "skills": {"required": True}, - } - - _attribute_map = { - "name": {"key": "name", "type": "str"}, - "description": {"key": "description", "type": "str"}, - "skills": {"key": "skills", "type": "[SearchIndexerSkill]"}, - "cognitive_services_account": { - "key": "cognitiveServices", - "type": "CognitiveServicesAccount", - }, - "knowledge_store": { - "key": "knowledgeStore", - "type": "SearchIndexerKnowledgeStore", - }, - "index_projections": { - "key": "indexProjections", - "type": "SearchIndexerIndexProjections", - }, - "e_tag": {"key": "@odata\\.etag", "type": "str"}, - "encryption_key": { - "key": "encryptionKey", - "type": "SearchResourceEncryptionKey", - }, - } - def __init__( self, *, @@ -106,7 +76,7 @@ def __init__( encryption_key: Optional["SearchResourceEncryptionKey"] = None, **kwargs: Any ) -> None: - super().__init__(**kwargs) + # pylint:disable=unused-argument self.name = name self.description = description self.skills = skills @@ -151,6 +121,26 @@ def _from_generated(cls, skillset) -> "SearchIndexerSkillset": kwargs["skills"] = custom_skills return cls(**kwargs) + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_generated().serialize(keep_readonly=keep_readonly, **kwargs) # type: ignore + + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "SearchIndexerSkillset": + """Parse a str using the RestAPI syntax and return a SearchIndexerSkillset instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A SearchIndexerSkillset instance + :rtype: SearchIndexerSkillset + :raises: DeserializationError if something went wrong + """ + return cls._from_generated(_SearchIndexerSkillset.deserialize(data, content_type=content_type)) + class EntityRecognitionSkillVersion(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Specifies the Entity Recognition skill version to use.""" @@ -385,7 +375,7 @@ def _from_generated(cls, skill): return None -class AnalyzeTextOptions(_serialization.Model): +class AnalyzeTextOptions: """Specifies some text and analysis components used to break that text into tokens. All required parameters must be populated in order to send to Azure. @@ -426,21 +416,7 @@ class AnalyzeTextOptions(_serialization.Model): :vartype char_filters: list[str] """ - _validation = { - "text": {"required": True}, - } - - _attribute_map = { - "text": {"key": "text", "type": "str"}, - "analyzer_name": {"key": "analyzerName", "type": "str"}, - "tokenizer_name": {"key": "tokenizerName", "type": "str"}, - "normalizer_name": {"key": "normalizerName", "type": "str"}, - "token_filters": {"key": "tokenFilters", "type": "[str]"}, - "char_filters": {"key": "charFilters", "type": "[str]"}, - } - def __init__(self, **kwargs): - super(AnalyzeTextOptions, self).__init__(**kwargs) self.text = kwargs["text"] self.analyzer_name = kwargs.get("analyzer_name", None) self.tokenizer_name = kwargs.get("tokenizer_name", None) @@ -458,6 +434,37 @@ def _to_analyze_request(self): char_filters=self.char_filters, ) + @classmethod + def _from_analyze_request(cls, analyze_request) -> "AnalyzeTextOptions": + return cls( + text=analyze_request.text, + analyzer_name=analyze_request.analyzer, + tokenizer_name=analyze_request.tokenizer, + normalizer_name=analyze_request.normalizer, + token_filters=analyze_request.token_filters, + char_filters=analyze_request.char_filters, + ) + + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_analyze_request().serialize(keep_readonly=keep_readonly, **kwargs) # type: ignore + + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "AnalyzeTextOptions": + """Parse a str using the RestAPI syntax and return a AnalyzeTextOptions instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A AnalyzeTextOptions instance + :rtype: AnalyzeTextOptions + :raises: DeserializationError if something went wrong + """ + return cls._from_analyze_request(AnalyzeRequest.deserialize(data, content_type=content_type)) + class CustomAnalyzer(LexicalAnalyzer): """Allows you to take control over the process of converting text into indexable/searchable tokens. @@ -672,7 +679,7 @@ def _from_generated(cls, pattern_tokenizer): ) -class SearchResourceEncryptionKey(_serialization.Model): +class SearchResourceEncryptionKey: """A customer-managed encryption key in Azure Key Vault. Keys that you create and manage can be used to encrypt or decrypt data-at-rest in Azure Cognitive Search, such as indexes and synonym maps. @@ -696,22 +703,7 @@ class SearchResourceEncryptionKey(_serialization.Model): :vartype application_secret: str """ - _validation = { - "key_name": {"required": True}, - "key_version": {"required": True}, - "vault_uri": {"required": True}, - } - - _attribute_map = { - "key_name": {"key": "keyVaultKeyName", "type": "str"}, - "key_version": {"key": "keyVaultKeyVersion", "type": "str"}, - "vault_uri": {"key": "keyVaultUri", "type": "str"}, - "application_id": {"key": "applicationId", "type": "str"}, - "application_secret": {"key": "applicationSecret", "type": "str"}, - } - def __init__(self, **kwargs): - super(SearchResourceEncryptionKey, self).__init__(**kwargs) self.key_name = kwargs["key_name"] self.key_version = kwargs["key_version"] self.vault_uri = kwargs["vault_uri"] @@ -751,8 +743,29 @@ def _from_generated(cls, search_resource_encryption_key): application_secret=application_secret, ) + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_generated().serialize(keep_readonly=keep_readonly, **kwargs) # type: ignore + + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "SearchResourceEncryptionKey": + """Parse a str using the RestAPI syntax and return a SearchResourceEncryptionKey instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A SearchResourceEncryptionKey instance + :raises: DeserializationError if something went wrong + """ + return cls._from_generated( # type: ignore + _SearchResourceEncryptionKey.deserialize(data, content_type=content_type) + ) + -class SynonymMap(_serialization.Model): +class SynonymMap: """Represents a synonym map definition. Variables are only populated by the server, and will be ignored when sending a request. @@ -780,27 +793,9 @@ class SynonymMap(_serialization.Model): :vartype e_tag: str """ - _validation = { - "name": {"required": True}, - "format": {"required": True, "constant": True}, - "synonyms": {"required": True}, - } - - _attribute_map = { - "name": {"key": "name", "type": "str"}, - "format": {"key": "format", "type": "str"}, - "synonyms": {"key": "synonyms", "type": "[str]"}, - "encryption_key": { - "key": "encryptionKey", - "type": "SearchResourceEncryptionKey", - }, - "e_tag": {"key": "@odata\\.etag", "type": "str"}, - } - format = "solr" def __init__(self, **kwargs): - super(SynonymMap, self).__init__(**kwargs) self.name = kwargs["name"] self.synonyms = kwargs["synonyms"] self.encryption_key = kwargs.get("encryption_key", None) @@ -826,8 +821,28 @@ def _from_generated(cls, synonym_map) -> "SynonymMap": e_tag=synonym_map.e_tag, ) + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_generated().serialize(keep_readonly=keep_readonly, **kwargs) # type: ignore -class SearchIndexerDataSourceConnection(_serialization.Model): + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "SynonymMap": + """Parse a str using the RestAPI syntax and return a SynonymMap instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A SynonymMap instance + :rtype: SynonymMap + :raises: DeserializationError if something went wrong + """ + return cls._from_generated(_SynonymMap.deserialize(data, content_type=content_type)) + + +class SearchIndexerDataSourceConnection: """Represents a datasource connection definition, which can be used to configure an indexer. All required parameters must be populated in order to send to Azure. @@ -862,37 +877,7 @@ class SearchIndexerDataSourceConnection(_serialization.Model): :vartype encryption_key: ~azure.search.documents.indexes.models.SearchResourceEncryptionKey """ - _validation = { - "name": {"required": True}, - "type": {"required": True}, - "connection_string": {"required": True}, - "container": {"required": True}, - } - - _attribute_map = { - "name": {"key": "name", "type": "str"}, - "description": {"key": "description", "type": "str"}, - "type": {"key": "type", "type": "str"}, - "connection_string": {"key": "connectionString", "type": "str"}, - "container": {"key": "container", "type": "SearchIndexerDataContainer"}, - "data_change_detection_policy": { - "key": "dataChangeDetectionPolicy", - "type": "DataChangeDetectionPolicy", - }, - "data_deletion_detection_policy": { - "key": "dataDeletionDetectionPolicy", - "type": "DataDeletionDetectionPolicy", - }, - "encryption_key": { - "key": "encryptionKey", - "type": "SearchResourceEncryptionKey", - }, - "e_tag": {"key": "@odata\\.etag", "type": "str"}, - "identity": {"key": "identity", "type": "SearchIndexerDataIdentity"}, - } - def __init__(self, **kwargs): - super(SearchIndexerDataSourceConnection, self).__init__(**kwargs) self.name = kwargs["name"] self.description = kwargs.get("description", None) self.type = kwargs["type"] @@ -941,6 +926,26 @@ def _from_generated(cls, search_indexer_data_source) -> "SearchIndexerDataSource identity=search_indexer_data_source.identity, ) + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> MutableMapping[str, Any]: + """Return the JSON that would be sent to server from this model. + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + return self._to_generated().serialize(keep_readonly=keep_readonly, **kwargs) # type: ignore + + @classmethod + def deserialize(cls, data: Any, content_type: Optional[str] = None) -> "SearchIndexerDataSourceConnection": + """Parse a str using the RestAPI syntax and return a SearchIndexerDataSourceConnection instance. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: A SearchIndexerDataSourceConnection instance + :rtype: SearchIndexerDataSourceConnection + :raises: DeserializationError if something went wrong + """ + return cls._from_generated(_SearchIndexerDataSource.deserialize(data, content_type=content_type)) + def pack_analyzer(analyzer): if not analyzer: diff --git a/sdk/search/azure-search-documents/tests/test_serialization.py b/sdk/search/azure-search-documents/tests/test_serialization.py new file mode 100644 index 000000000000..44a1d81f2a0c --- /dev/null +++ b/sdk/search/azure-search-documents/tests/test_serialization.py @@ -0,0 +1,79 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.search.documents.indexes.models import ( + SearchIndex, + SearchIndexerSkillset, + SearchFieldDataType, + SimpleField, + SearchableField, + ComplexField, + ScoringProfile, + CorsOptions, + CognitiveServicesAccountKey, + InputFieldMappingEntry, + OutputFieldMappingEntry, + SearchIndexerSkillset, + SplitSkill, + TextSplitMode, +) + + +def test_serialize_search_index(): + new_index_name = "hotels" + fields = [ + SimpleField(name="hotelId", type=SearchFieldDataType.String, key=True), + SimpleField(name="baseRate", type=SearchFieldDataType.Double), + SearchableField(name="description", type=SearchFieldDataType.String, collection=True), + SearchableField(name="hotelName", type=SearchFieldDataType.String), + ComplexField( + name="address", + fields=[ + SimpleField(name="streetAddress", type=SearchFieldDataType.String), + SimpleField(name="city", type=SearchFieldDataType.String), + SimpleField(name="state", type=SearchFieldDataType.String), + ], + collection=True, + ), + ] + cors_options = CorsOptions(allowed_origins=["*"], max_age_in_seconds=60) + scoring_profile = ScoringProfile(name="MyProfile") + scoring_profiles = [] + scoring_profiles.append(scoring_profile) + index = SearchIndex( + name=new_index_name, fields=fields, scoring_profiles=scoring_profiles, cors_options=cors_options + ) + search_index_serialized = index.serialize() + search_index = SearchIndex.deserialize(search_index_serialized) + assert search_index + + +def test_serialize_search_indexer_skillset(): + COGNITIVE_KEY = ... + COGNITIVE_DESCRIPTION = ... + + cognitive_services_account = CognitiveServicesAccountKey(key=COGNITIVE_KEY, description=COGNITIVE_DESCRIPTION) + + inputs = [InputFieldMappingEntry(name="text", source="/document/content")] + + outputs = [OutputFieldMappingEntry(name="textItems", target_name="pages")] + + split_skill = SplitSkill( + name="SplitSkill", + inputs=inputs, + outputs=outputs, + context="/document", + text_split_mode=TextSplitMode.PAGES, + maximum_page_length=5000, + ) + + skills = [split_skill] + skillset = SearchIndexerSkillset( + name="Skillset", skills=skills, cognitive_services_account=cognitive_services_account + ) + + serialized_skillset = skillset.serialize() + skillset = SearchIndexerSkillset.deserialize(serialized_skillset) + assert skillset