Merge pull request #114 from singnet/SPS-18

Optimization, rework and refactor

Author: Arondondon

snet/sdk/__init__.py

@@ -9,10 +9,11 @@
 import google.protobuf.internal.api_implementation
 from google.protobuf import symbol_database as _symbol_database
 
-from snet.sdk.registry.models import StorageType
+from snet.sdk.exceptions import NoGroupsFoundError, GroupNotFoundError, ServiceMetadataMismatchError
+from snet.sdk.registry.models import StorageType, FileURI
 from snet.sdk.registry.organization_metadata import OrganizationMetadata
 from snet.sdk.registry.registry_contract import RegistryContract
-from snet.sdk.registry.service_metadata import MPEServiceMetadata, ServiceMetadata
+from snet.sdk.registry.service_metadata import ServiceMetadata, Group
 
 with warnings.catch_warnings():
     # Suppress the eth-typing package`s warnings related to some new networks
@@ -74,15 +75,18 @@ def create_service_client(
         options=None,
         concurrent_calls: int = 1,
     ):
-
-        # Create and instance of the Config object,
-        # so we can create an instance of ClientLibGenerator
+        service_metadata = self._enhance_service_metadata(org_id, service_id)
         lib_generator = ClientLibGenerator(self.storage_provider, org_id, service_id)
 
-        # Download the proto file and generate stubs if needed
+        if service_metadata.service_api_source is not None:
+            service_api_source = service_metadata.service_api_source
+        else:
+            service_api_source = service_metadata.model_ipfs_hash
+        service_api_source = FileURI.from_raw_uri(service_api_source)
+
         force_update = config.FORCE_UPDATE
         if force_update:
-            lib_generator.generate_client_library()
+            lib_generator.generate_client_library(service_api_source)
         else:
             path_to_pb_files = lib_generator.proto_dir
             pb_2_file_name = find_file_by_keyword(
@@ -93,7 +97,7 @@ def create_service_client(
             )
             if not pb_2_file_name or not pb_2_grpc_file_name:
                 print("Generating client library...")
-                lib_generator.generate_client_library()
+                lib_generator.generate_client_library(service_api_source)
 
         if options is None:
             options = dict()
@@ -103,16 +107,14 @@ def create_service_client(
         if payment_strategy is None:
             payment_strategy = payment_strategy_type.value()
 
-        service_metadata = self._enhance_service_metadata(org_id, service_id)
-        group = self._get_service_group_details(service_metadata, group_name)
+        group = self._get_service_group(org_id, service_id, service_metadata, group_name)
 
         service_stubs = self.get_service_stub(lib_generator)
 
         pb2_module = self.get_module_by_keyword("pb2.py", lib_generator)
         _service_client = ServiceClient(
             org_id,
             service_id,
-            service_metadata,
             group,
             service_stubs,
             payment_strategy,
@@ -162,36 +164,28 @@ def get_module_by_keyword(self, keyword: str, lib_generator: ClientLibGenerator)
         module_name = os.path.splitext(file_name)[0]
         return ModuleName(module_name)
 
-    def get_service_metadata(self, org_id, service_id):
+    def get_service_metadata(self, org_id, service_id) -> ServiceMetadata:
         service = self.registry_contract.get_service(org_id, service_id)
         return self.storage_provider.fetch_service_metadata(service.metadata_uri)
 
     def get_organization_metadata(self, org_id: str) -> OrganizationMetadata:
         org = self.registry_contract.get_org(org_id)
         return self.storage_provider.fetch_org_metadata(org.metadata_uri)
 
-    def _get_first_group(self, service_metadata: MPEServiceMetadata) -> dict:
-        return service_metadata["groups"][0]
-
-    def _get_group_by_group_name(
-        self, service_metadata: MPEServiceMetadata, group_name: str
-    ) -> dict:
-        for group in service_metadata["groups"]:
-            if group["group_name"] == group_name:
-                return group
-        # TODO: configure exceptions
-        raise Exception()
-
-    def _get_service_group_details(
-        self, service_metadata: MPEServiceMetadata, group_name: str
-    ) -> dict:
-        if len(service_metadata["groups"]) == 0:
-            raise Exception("No Groups found for given service, Please add group to the service")
+    def _get_service_group(
+        self, org_id: str, service_id: str, service_metadata: ServiceMetadata, group_name: str
+    ) -> Group:
+        if len(service_metadata.groups) == 0:
+            raise NoGroupsFoundError(org_id, service_id)
 
         if group_name is None:
-            return self._get_first_group(service_metadata)
+            return service_metadata.groups[0]
 
-        return self._get_group_by_group_name(service_metadata, group_name)
+        for group in service_metadata.groups:
+            if group.group_name == group_name:
+                return group
+
+        raise GroupNotFoundError(org_id, service_id, group_name)
 
     def get_organization_list(self) -> list:
         return self.registry_contract.list_orgs()
@@ -215,13 +209,68 @@ def publish_service_comprehensively(
         5. publish service into Registry contract
         """
         proto_uri = self.storage_provider.publish_proto(proto_dir, storage_type)
-
         metadata.service_api_source = str(proto_uri)
         metadata.mpe_address = self.mpe_contract.contract.address
 
+        self._check_and_update_service_groups(org_id, metadata.groups)
         metadata_uri = self.storage_provider.publish_service_metadata(metadata, storage_type)
+
         receipt = self.registry_contract.create_service(
             self.account, org_id, service_id, metadata_uri
         )
 
         return receipt["status"] != 0
+
+    def update_service(
+        self,
+        org_id: str,
+        service_id: str,
+        metadata: ServiceMetadata,
+        proto_dir: Union[str, Path, None] = None,
+        storage_type: StorageType = StorageType.IPFS,
+    ) -> bool:
+        if proto_dir is not None:
+            proto_uri = self.storage_provider.publish_proto(proto_dir, storage_type)
+            metadata.service_api_source = str(proto_uri)
+
+        if not metadata.mpe_address:
+            metadata.mpe_address = self.mpe_contract.contract.address
+
+        self._check_and_update_service_groups(org_id, metadata.groups)
+        metadata_uri = self.storage_provider.publish_service_metadata(metadata, storage_type)
+
+        receipt = self.registry_contract.update_service_metadata(
+            self.account, org_id, service_id, metadata_uri
+        )
+
+        return receipt["status"] != 0
+
+    def update_organization(
+        self,
+        org_id: str,
+        organization_metadata: OrganizationMetadata,
+        storage_type: StorageType = StorageType.IPFS,
+    ) -> bool:
+        metadata_uri = self.storage_provider.publish_organization_metadata(
+            organization_metadata, storage_type
+        )
+        receipt = self.registry_contract.update_org_metadata(self.account, org_id, metadata_uri)
+
+        return receipt["status"] != 0
+
+    def _check_and_update_service_groups(
+        self, org_id: str, service_groups: list[Group]
+    ) -> list[Group]:
+        org = self.registry_contract.get_org(org_id)
+        org_metadata = self.storage_provider.fetch_org_metadata(org.metadata_uri)
+        org_groups_map = {g.group_name: g for g in org_metadata.groups}
+
+        for group in service_groups:
+            try:
+                group.group_id = org_groups_map[group.group_name].group_id
+            except KeyError:
+                raise ServiceMetadataMismatchError(
+                    "All groups added to the service must also exist in the organization!"
+                )
+
+        return service_groups

snet/sdk/client_lib_generator.py

@@ -1,6 +1,7 @@
 import os
 from pathlib import Path
 
+from snet.sdk.registry.models import FileURI
 from snet.sdk.registry.storage_provider import StorageProvider
 from snet.sdk.utils.utils import compile_proto
 
@@ -20,9 +21,9 @@ def __init__(
         self.proto_dir: Path = proto_dir if proto_dir else Path.home().joinpath(".snet")
         self.generate_directories_by_params()
 
-    def generate_client_library(self) -> None:
+    def generate_client_library(self, service_api_source: FileURI) -> None:
         try:
-            self.receive_proto_files()
+            self.receive_proto_files(service_api_source)
             compilation_result = compile_proto(
                 entry_path=self.proto_dir,
                 codegen_dir=self.proto_dir,
@@ -35,8 +36,8 @@ def generate_client_library(self) -> None:
                     f'in org with id "{self.org_id}" '
                     f"generated at {self.proto_dir}"
                 )
-        except Exception as e:
-            print(str(e))
+        except Exception:
+            raise Exception("Error while proto compilation!")
 
     def generate_directories_by_params(self) -> None:
         if not self.proto_dir.is_absolute():
@@ -47,13 +48,7 @@ def create_service_client_libraries_path(self) -> None:
         self.proto_dir = self.proto_dir.joinpath(self.org_id, self.service_id, self.language)
         self.proto_dir.mkdir(parents=True, exist_ok=True)
 
-    def receive_proto_files(self) -> None:
-        metadata = self._metadata_provider.fetch_service_metadata(
-            org_id=self.org_id, service_id=self.service_id
-        )
-        service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash")
-
-        # Receive proto files
+    def receive_proto_files(self, service_api_source: FileURI) -> None:
         if self.proto_dir.exists():
             self._metadata_provider.fetch_and_extract_proto(service_api_source, self.proto_dir)
         else:

snet/sdk/exceptions.py

@@ -1,7 +1,15 @@
+from grpc import RpcError
+
+
 class SnetSDKError(Exception):
+    """Base SNET SDK exception class"""
+
     pass
 
 
+# ==================== Blockchain interaction Errors ====================
+
+
 class ContractError(SnetSDKError):
     pass
 
@@ -27,7 +35,7 @@ def __init__(self, tx_hash: str, event_name: str):
         )
 
 
-class RegistryContractError(ContractError):
+class RegistryContractError(ValueError, ContractError):
     pass
 
 
@@ -41,18 +49,28 @@ def __init__(self, org_id: str, service_id: str):
         super().__init__(f"Service with org_id={org_id} service_id={service_id} doesn't exist!")
 
 
-class UnauthorizedCallerError(TransactionError):
+class UnauthorizedCallerError(RegistryContractError):
     pass
 
 
 class UnauthorizedOrgMemberError(UnauthorizedCallerError):
     def __init__(self, org_id: str, address: str):
-        super().__init__(f"Address {address} isn't owner or member of the organization {org_id}!")
+        super().__init__(
+            f"Address {address} isn't an owner or a member of the organization {org_id}!"
+        )
 
 
-class UnsupportedStorageTypeError(ValueError, SnetSDKError):
-    def __init__(self, storage_type: str):
-        super().__init__(f"Unsupported storage type: {storage_type}!")
+class UnauthorizedOrgOwnerError(UnauthorizedCallerError):
+    def __init__(self, org_id: str, address: str):
+        super().__init__(f"Address {address} isn't an owner of the organization {org_id}!")
+
+
+class IncorrectWalletAddressError(RegistryContractError):
+    def __init__(self, address: str):
+        super().__init__(f"Address {address} is not a correct Ethereum address!")
+
+
+# ==================== Metadata Errors ====================
 
 
 class MetadataMismatchError(ValueError, SnetSDKError):
@@ -67,18 +85,40 @@ class OrganizationMetadataMismatchError(MetadataMismatchError):
     pass
 
 
+# ==================== Storage Provider Errors ====================
+
+
 class StorageProviderError(SnetSDKError):
     pass
 
 
+class UnsupportedStorageTypeError(ValueError, StorageProviderError):
+    def __init__(self, storage_type: str):
+        super().__init__(f"Unsupported storage type: {storage_type}!")
+
+
 class LighthouseError(StorageProviderError):
     def __init__(self):
         super().__init__(
             "Lighthouse internal error! Most likely, you did not specify LIGHTHOUSE_TOKEN in the config or it expired."
         )
 
 
-class PublishProtoError(StorageProviderError, ValueError):
+class IPFSError(StorageProviderError):
+    pass
+
+
+class IPFSHashMismatchError(IPFSError):
+    def __init__(self):
+        super().__init__("IPFS hash mismatch with data")
+
+
+class IPFSHashCheckError(IPFSError):
+    def __init__(self):
+        super().__init__("IPFS hash integrity check failed!")
+
+
+class PublishProtoError(ValueError, StorageProviderError):
     pass
 
 
@@ -90,3 +130,64 @@ def __init__(self, dir_path: str):
 class ProtoFilesNotFoundError(PublishProtoError):
     def __init__(self, dir_path: str):
         super().__init__(f"Cannot find any .proto file in {dir_path}!")
+
+
+class ExtractingProtoError(ValueError, StorageProviderError):
+    pass
+
+
+# ==================== Training Errors ====================
+
+
+class TrainingError(SnetSDKError):
+    pass
+
+
+class WrongDatasetError(TrainingError):
+    def __init__(self, errors: list[str]):
+        self.errors = errors
+        exception_msg = "Dataset check failed:\n"
+        for check in errors:
+            exception_msg += f"\t{check}\n"
+        super().__init__(exception_msg)
+
+
+class WrongMethodError(ValueError, TrainingError):
+    def __init__(self, method_name: str):
+        super().__init__(f"Method with name {method_name} not found!")
+
+
+class NoTrainingError(ValueError, TrainingError):
+    def __init__(self, org_id: str, service_id: str):
+        super().__init__(
+            f"Training is not implemented for the service with org_id={org_id} and service_id={service_id}!"
+        )
+
+
+class GRPCError(RpcError, TrainingError):
+    def __init__(self, error: RpcError):
+        super().__init__(f"An error occurred during the grpc call: {error}.")
+
+
+class NoSuchModelError(ValueError, TrainingError):
+    def __init__(self, model_id: str):
+        super().__init__(f"Model with id {model_id} not found!")
+
+
+# ==================== Service Client Errors ====================
+
+
+class ServiceClientError(SnetSDKError):
+    pass
+
+
+class NoGroupsFoundError(ServiceClientError):
+    def __init__(self, org_id: str, service_id: str):
+        super().__init__(f"Service with org_id={org_id} service_id={service_id} has no groups!")
+
+
+class GroupNotFoundError(ServiceClientError):
+    def __init__(self, org_id: str, service_id: str, group_name: str):
+        super().__init__(
+            f"Service with org_id={org_id} service_id={service_id} has no group with group_name={group_name}!"
+        )