diff --git a/tools/abel/abel.py b/tools/abel/abel.py index b88eddfd43..eeebb60824 100644 --- a/tools/abel/abel.py +++ b/tools/abel/abel.py @@ -21,21 +21,28 @@ def main(parsed_args: ParseArgs) -> None: + print( '\nHow would you like to use ABEL?\n' - + '1: Modify a spreadsheet/building config for an existing building\n' - + '2: Create a spreadsheet for a new building\n' - + '3: Split a building config\n' + + '1: Create a building config yaml file from a spreadsheet.\n' + + '2: Create a spreadsheet from a building config.\n' + 'q: quit\n' ) function_choice = input('Please select an option: ') - new_workflow = Workflow(parsed_args) + new_workflow = Workflow( + parsed_args.credential, + parsed_args.modified_types_filepath, + parsed_args.output_dir + ) if function_choice == '1': - new_workflow.UpdateWorkflow() + new_workflow.SpreadsheetWorkflow( + parsed_args.spreadsheet_id, + parsed_args.building_config, + parsed_args.subscription, + parsed_args.timeout + ) elif function_choice == '2': - new_workflow.InitWorkflow() - elif function_choice == '3': - new_workflow.SplitWorkflow() + new_workflow.ConfigWorkflow(parsed_args.building_config) elif function_choice == 'q': print('Bye bye') sys.exit() diff --git a/tools/abel/model/export_helper.py b/tools/abel/model/export_helper.py index e93ddd6ecc..8d09ee0ffd 100644 --- a/tools/abel/model/export_helper.py +++ b/tools/abel/model/export_helper.py @@ -13,6 +13,7 @@ # limitations under the License. """Helper module for exporting a valid Building Configuration or spreadsheet.""" +import enum from typing import Any, Dict, List, Optional # pylint: disable=g-importing-member @@ -25,7 +26,6 @@ from model.constants import CONFIG_CODE from model.constants import CONFIG_CONNECTIONS from model.constants import CONFIG_ETAG -from model.constants import CONFIG_INITIALIZE from model.constants import CONFIG_LINKS from model.constants import CONFIG_METADATA from model.constants import CONFIG_OPERATION @@ -51,6 +51,13 @@ from model.model_error import SpreadsheetAuthorizationError from validate.field_translation import FieldTranslation +class ConfigMode(enum.Enum): + + # config mode for a building config being updated. + UPDATE = 'UPDATE' + + # config mode for a building being initialized. + INITIALIZE = 'INITIALIZE' class GoogleSheetExport(object): """Class to help write ABEL data types to a Google Sheets spreadsheet. @@ -112,8 +119,8 @@ def __init__(self, model: Model): """ self.model = model - def ExportUpdateBuildingConfiguration( - self, filepath: str, operations: List[EntityOperation] + def ExportBuildingConfiguration( + self, filepath: str, operations: List[EntityOperation] = None ) -> Dict[str, Any]: """Exports a building Config under the UPDATE operation. @@ -126,14 +133,25 @@ def ExportUpdateBuildingConfiguration( Dictionary mapping of a building config under the update operation. """ site = self.model.site - entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: 'UPDATE'}} - for operation in operations: - entity = operation.entity + config_mode = ConfigMode.UPDATE + operations_map = {} + if not site.etag: + config_mode = ConfigMode.INITIALIZE + entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: config_mode.value}} + if not operations: + iterator = [self.model.guid_to_entity_map.GetEntityByGuid(entity_guid) + for entity_guid in site.entities] + else: + iterator = [operation.entity for operation in operations] + operations_map = {operation.entity.bc_guid: operation for operation in + operations} + for entity in iterator: + operation = operations_map.get(entity.bc_guid) if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( - entity, operation + entity, config_mode, operation ) } ) @@ -141,7 +159,7 @@ def ExportUpdateBuildingConfiguration( entity_yaml_dict.update( { entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( - entity, operation + entity, config_mode, operation ) } ) @@ -151,10 +169,11 @@ def ExportUpdateBuildingConfiguration( site.guid: { CONFIG_CODE: site.code, CONFIG_TYPE: site.namespace + '/' + site.type_name, - CONFIG_ETAG: site.etag, } } ) + if site.etag: + entity_yaml_dict.get(site.guid).update({CONFIG_ETAG: site.etag}) try: with open(filepath, WRITE, encoding=UTF_8) as file: for key, value in entity_yaml_dict.items(): @@ -165,56 +184,6 @@ def ExportUpdateBuildingConfiguration( return entity_yaml_dict # TODO(b/233756557) Allow user to set config_metadata operation. - def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: - """Exports an ABEL concrete model graph to a Building Config file. - - Args: - filepath: Absolute export path for a Building Config. - - Returns: - A dictionary model of the exported building config. - - Raises: - PermissionError: When ABEL is denied access to filepath. - """ - site = self.model.site - entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: CONFIG_INITIALIZE}} - for entity_guid in site.entities: - entity = self.model.guid_to_entity_map.GetEntityByGuid(entity_guid) - if isinstance(entity, ReportingEntity): - entity_yaml_dict.update( - { - entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( - entity=entity, - operation=None, - ) - } - ) - elif isinstance(entity, VirtualEntity): - entity_yaml_dict.update( - { - entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( - entity=entity, operation=None - ) - } - ) - - entity_yaml_dict.update( - { - site.guid: { - CONFIG_CODE: site.code, - CONFIG_TYPE: site.namespace + '/' + site.type_name, - } - } - ) - try: - with open(filepath, WRITE, encoding=UTF_8) as file: - for key, value in entity_yaml_dict.items(): - file.write(as_document({key: value}).as_yaml()) - file.write('\n') - except PermissionError: - print(f'Permission denied when writing to {filepath}') - return entity_yaml_dict def _AddOperationToBlock( self, operation: EntityOperation @@ -229,13 +198,18 @@ def _AddOperationToBlock( return update_dict def _GetReportingEntityBuildingConfigBlock( - self, entity: ReportingEntity, operation: Optional[EntityOperation] + self, + entity: ReportingEntity, + config_mode: ConfigMode, + operation: Optional[EntityOperation] ) -> Dict[str, object]: """Returns a Building Config formatted reporting entity block dictionary. Args: entity: A ReportingEntity instance. - operation: + operation: The operation acting on the entity... + config_mode: ConfigMode instance representing a building config's metadata + config mode. Returns: A dictionary in Building Config format ready to be parsed into yaml. @@ -273,12 +247,16 @@ def _GetReportingEntityBuildingConfigBlock( reporting_entity_yaml.update( {CONFIG_TYPE: entity.namespace.value + '/' + str(entity.type_name)} ) - if operation: + if operation and not operation.operation == EntityOperationType.EXPORT and \ + config_mode == ConfigMode.UPDATE: reporting_entity_yaml.update(self._AddOperationToBlock(operation)) return reporting_entity_yaml def _GetVirtualEntityBuildingConfigBlock( - self, entity: VirtualEntity, operation: Optional[EntityOperation] + self, + entity: VirtualEntity, + config_mode: ConfigMode, + operation: Optional[EntityOperation] ) -> Dict[str, object]: """Returns a Building Config formatted virtual entity block dictionary. @@ -298,7 +276,8 @@ def _GetVirtualEntityBuildingConfigBlock( virtual_entity_yaml.update( {CONFIG_TYPE: entity.namespace.value + '/' + str(entity.type_name)} ) - if operation: + if operation and not operation.operation == EntityOperationType.EXPORT and \ + config_mode == ConfigMode.UPDATE: virtual_entity_yaml.update(self._AddOperationToBlock(operation)) return virtual_entity_yaml diff --git a/tools/abel/model/from_spreadsheet.py b/tools/abel/model/from_spreadsheet.py index 95cbb6e480..e8d52809a9 100644 --- a/tools/abel/model/from_spreadsheet.py +++ b/tools/abel/model/from_spreadsheet.py @@ -131,7 +131,8 @@ def LoadStatesFromSpreadsheet( state_entry[REPORTING_ENTITY_GUID] = guid_to_entity_map.GetEntityGuidByCode( state_entry[REPORTING_ENTITY_CODE] ) - states.append(State.FromDict(states_dict=state_entry)) + new_state = State.FromDict(states_dict=state_entry) + states.append(new_state) return states @@ -172,7 +173,7 @@ def LoadConnectionsFromSpreadsheet( def LoadOperationsFromSpreadsheet( entity_entries: Dict[str, str], guid_to_entity_map: GuidToEntityMap ) -> List[EntityOperation]: - """loads a list of entity dicitionary mappings into EntityOperation instances. + """loads a list of entity dictionary mappings into EntityOperation instances. Args: entity_entries: A list of Python Dictionaries mapping entity attributes @@ -192,8 +193,10 @@ def LoadOperationsFromSpreadsheet( new_entity.bc_guid = str(uuid.uuid4()) guid_to_entity_map.AddEntity(new_entity) operation = entity_entry.get(OPERATION) - if not operation: + if not operation and new_entity.etag: operation = EntityOperationType.EXPORT + elif not operation and not new_entity.etag: + operation = EntityOperationType.ADD operation_instance = EntityOperation( entity=new_entity, operation=EntityOperationType(operation) ) diff --git a/tools/abel/model/model_builder.py b/tools/abel/model/model_builder.py index 2b55d55fe5..66e298de8c 100644 --- a/tools/abel/model/model_builder.py +++ b/tools/abel/model/model_builder.py @@ -14,6 +14,7 @@ """Helper module for concrete model construction.""" import datetime +import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -214,8 +215,9 @@ def Build(self) -> ...: # For each entity, Add connections where entity is the source for guid in self.site.entities: entity = self.guid_to_entity_map.GetEntityByGuid(guid) + guid = uuid.UUID(guid) for connection in self.connections: - if connection.target_entity_guid == guid: + if uuid.UUID(connection.target_entity_guid) == guid: entity.AddConnection(connection) # For each field in the model for field in self.fields: diff --git a/tools/abel/model/model_helper.py b/tools/abel/model/model_helper.py index 637f7136eb..964128a5ce 100644 --- a/tools/abel/model/model_helper.py +++ b/tools/abel/model/model_helper.py @@ -181,6 +181,14 @@ def DetermineEntityOperations( operation which is being performed on it. """ operations = [] + if not current_model: + for entity in updated_model.entities: + if not entity.etag: + operations.append(entity_operation.EntityOperation( + entity, + operation=entity_enumerations.EntityOperationType.ADD + )) + return operations for import_entity in updated_model.entities: if import_entity.bc_guid not in set( entity.bc_guid for entity in current_model.entities diff --git a/tools/abel/model/workflow.py b/tools/abel/model/workflow.py index faa39417d1..15d6fcaba1 100644 --- a/tools/abel/model/workflow.py +++ b/tools/abel/model/workflow.py @@ -14,12 +14,13 @@ """Module for various ABEL workflows..""" import os +import sys from typing import List, Optional +import warnings import webbrowser # pylint: disable=g-importing-member from model import authenticator -from model import entity_enumerations from model import export_helper from model import import_helper from model import model_helper @@ -45,21 +46,13 @@ class Workflow(object): 3: Ingest an exported BC and write to spreadsheet that user can then modify. """ - def __init__(self, argset: ParseArgs): + def __init__(self, oauth_credential: str, modified_types_filepath: str = + None, output_dir: str = None): self.google_sheets_service = ( - authenticator.GetGoogleSheetsServiceByCredential(argset.credential) + authenticator.GetGoogleSheetsServiceByCredential(oauth_credential) ) - self.subscription = argset.subscription - self.spreadsheet_id = argset.spreadsheet_id - if not argset.building_config: - self.bc_filepath = None - else: - self.bc_filepath = os.path.expanduser(argset.building_config) - self.timeout = argset.timeout - self.modified_types_filepath = argset.modified_types_filepath - self.output_dir = argset.output_dir - self.bc_model = None - self.ss_model = None + self.modified_types_filepath = modified_types_filepath + self.output_dir = output_dir def _ImportSpreadsheetAndBuildModel(self, spreadsheet_id) -> Model: """Reads a Google Sheets spreadsheet and builds an ABEL model. @@ -99,14 +92,15 @@ def _ImportSpreadsheetAndBuildModel(self, spreadsheet_id) -> Model: ) return unbuilt_model.Build(), operations - def _ImportBCAndBuildModel(self) -> Model: + def _ImportBCAndBuildModel(self, bc_filepath: str) -> Model: """Reads a local Building Config YAML file and returns and ABEL Model. Returns: An ABEL model instance and a list of EntityOperation instances. """ + warnings.simplefilter("ignore", UserWarning) imported_building_config = import_helper.DeserializeBuildingConfiguration( - filepath=self.bc_filepath + filepath=bc_filepath ) # STEP 2 @@ -119,7 +113,7 @@ def _ImportBCAndBuildModel(self) -> Model: def _ExportAndWriteToSpreadsheet( self, model: Model, operations: Optional[List[EntityOperation]] = None - ) -> None: + ) -> tuple[str, str]: """Exports an ABEL spreadsheet for a user to interact with. Args: @@ -146,6 +140,8 @@ def _ValidateAndExportBuildingConfig( self, model: Model, operations: list[EntityOperation] = None, + subscription: str = None, + timeout: str = None ) -> None: """Helper function for validating and exporting a building config. @@ -154,133 +150,82 @@ def _ValidateAndExportBuildingConfig( operations: [Optional] A list of EntityOperation instances for the model. """ bc_path = os.path.join(self.output_dir, EXPORT_BUILDING_CONFIG_NAME) - if operations: - export_helper.BuildingConfigExport( - model - ).ExportUpdateBuildingConfiguration( - filepath=bc_path, - operations=operations, - ) - else: - export_helper.BuildingConfigExport(model).ExportInitBuildingConfiguration( - filepath=bc_path - ) + export_helper.BuildingConfigExport( + model + ).ExportBuildingConfiguration( + filepath=bc_path, + operations=operations, + ) # Run instance validator - print('Validating Export.') + print('Validating building config yaml file export.') report_name = handler.RunValidation( filenames=[bc_path], report_directory=self.output_dir, modified_types_filepath=self.modified_types_filepath, default_types_filepath=ONTOLOGY_ROOT, - subscription=self.subscription, - timeout=self.timeout, + subscription=subscription, + timeout=timeout, ) print(f'Instance validator log: {report_name}') print(f'Exported Building Configuration: {bc_path}') - # TODO: b/296067948 - Ingest and export spreadsheet with update operations. - def UpdateWorkflow(self) -> None: - """Workflow for generating a building config with update operations. - - STEPS: - 1. Ingest a building config - 2. write to a spreadsheet - 3. Either let the user quit or build an update bc from updated spreadsheet - 4. If build update BC, ingest modified spreadsheet(same id) and bc(already - persisting) - 5. Create operations with two models. - 6. Write to building config. - """ - - print( - '\nHow would you like to use ABEL?\n' - + '1: Edit or update an existing building config\n' - + '2: Create building config from an updated spreadsheet\n' - + 'q: quit\n' - ) - function_choice = input('Please select an option: ') - if function_choice == '1': - # Export BC and build model - self.bc_model, bc_operations = self._ImportBCAndBuildModel() - spreadsheet_url, spreadsheet_id = self._ExportAndWriteToSpreadsheet( - self.bc_model, bc_operations - ) - - # Write to spreadsheet - webbrowser.open(spreadsheet_url) + def SpreadsheetWorkflow( + self, + spreadsheet_id: str, + bc_filepath: str = None, + gcp_subscription: str = None, + timeout: int = None + ) -> None: + """Workflow that Ingests a a spreadsheet and outputs a valid building + config. - # Wait for user to edit spreadsheet - input('Edit spreadsheet and press any key when done.') + If a building config path is provided to this workflow, then ABEL can + compare entities in a spreadsheet to entities in a building config, + and determine update operations. If no building config path is provided + then the spreadsheet will be treating as an init config. - # Build model from spreadsheet - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - spreadsheet_id=spreadsheet_id - ) - - # Determine entity operations on an updated model - generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model - ) - operations_list = model_helper.ReconcileOperations( - generated_operations=generated_operations, - model_operations=ss_operations, - ) + Args: + spreadsheet_id: Google Sheets spreadsheet ID. + bc_filepath: Relative or Absolute filepath to a building config yaml + file. + gcp_subscription: Path to a GCP PubSub subscription. + timeout: Timeout duration in seconds for listening to PubSub. + """ + # TODO: b/ + if bc_filepath: + bc_model, bc_operations = self._ImportBCAndBuildModel(bc_filepath) + else: + print('No building config input') + bc_model = None - # Finally export to building config - export_engine = export_helper.BuildingConfigExport(model=self.ss_model) - export_engine.ExportUpdateBuildingConfiguration( - filepath=os.path.join(self.output_dir, EXPORT_BUILDING_CONFIG_NAME), - operations=operations_list, - ) - elif function_choice == '2': - assert self.spreadsheet_id is not None and self.bc_filepath is not None - self.bc_model = self._ImportBCAndBuildModel()[0] - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - self.spreadsheet_id - ) - generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model - ) - operations_list = model_helper.ReconcileOperations( - generated_operations=generated_operations, - model_operations=ss_operations, - ) - self._ValidateAndExportBuildingConfig( - model=self.ss_model, - operations=operations_list, - ) + if not spreadsheet_id: + print('No spreadsheet id input.\nExiting ABEL...') + sys.exit(0) - def InitWorkflow(self) -> None: - """Workflow for generating a building config under the INITIALIZE operation. + ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( + spreadsheet_id + ) + generated_operations = model_helper.DetermineEntityOperations( + current_model=bc_model, updated_model=ss_model + ) + operations_list = model_helper.ReconcileOperations( + generated_operations=generated_operations, + model_operations=ss_operations, + ) + self._ValidateAndExportBuildingConfig( + model=ss_model, + operations=operations_list, + subscription=gcp_subscription, + timeout=timeout + ) - STEPS: - 1. Ingest and parse an ABEL spreadsheet. - 2. Export and validate a building config. - """ - if not self.spreadsheet_id: - self.spreadsheet_id = input( - 'Please provide a Google Sheets spreadsheet id.' - ) - self.ss_model = self._ImportSpreadsheetAndBuildModel(self.spreadsheet_id)[0] - self._ValidateAndExportBuildingConfig(model=self.ss_model) + def ConfigWorkflow(self, bc_filepath: str) -> None: + """Workflow to create a Google sheets spreadsheet from a building config""" - def SplitWorkflow(self) -> None: - """Workflow to take in a building config and split it. + bc_model, bc_operations = self._ImportBCAndBuildModel(bc_filepath) + spreadsheet_details = self._ExportAndWriteToSpreadsheet( + bc_model, bc_operations + ) - STEPS: - 1. ingest building config. - 2. Split building config. - 3. Export the split building config. - """ - if not self.bc_filepath: - self.bc_filepath = input('Please provide a building config file path.') - else: - namespace_string = input('Desired namespace: ') - namespace = entity_enumerations.EntityNamespace(namespace_string.upper()) - self.bc_model, bc_operations = self._ImportBCAndBuildModel() - split_model, split_operations = model_helper.Split( - self.bc_model, bc_operations, namespace - ) - self._ValidateAndExportBuildingConfig( - model=split_model, operations=split_operations - ) + # Write to spreadsheet + webbrowser.open(spreadsheet_details[0]) diff --git a/tools/abel/tests/export_helper_test.py b/tools/abel/tests/export_helper_test.py index 930de310b5..88d14aa9e4 100644 --- a/tools/abel/tests/export_helper_test.py +++ b/tools/abel/tests/export_helper_test.py @@ -150,7 +150,8 @@ def testWriteOneSheetRaisesSpreadsheetAuthorizationError(self): def testExportBuildingConfigExportsVirtualEntityKeysCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) expected_keys = ['code', 'etag', 'connections', 'links', 'type'] exported_keys = list( @@ -163,7 +164,8 @@ def testExportBuildingConfigExportsVirtualEntityKeysCorrectly(self): def testExportBuildingConfigExportsReportingEntityKeysCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) expected_keys = ['cloud_device_id', 'code', 'etag', 'translation', 'type'] exported_keys = list( @@ -174,7 +176,8 @@ def testExportBuildingConfigExportsReportingEntityKeysCorrectly(self): def testExportBuildingConfigExportsCloudDeviceIDAsString(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) expected_cdid = '2541901344105616' exported_cdid = exported_building_config.get('test_reporting_guid').get( @@ -186,7 +189,8 @@ def testExportBuildingConfigExportsCloudDeviceIDAsString(self): def testExportBuildingConfigExportsMultiStateValueFieldStates(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) multi_state_value_field_states = ( exported_building_config.get('test_reporting_guid') @@ -200,7 +204,8 @@ def testExportBuildingConfigExportsMultiStateValueFieldStates(self): def testExportBuildingConfigExportsUnitsCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) exported_units = ( exported_building_config.get('test_reporting_guid') @@ -215,7 +220,7 @@ def testExportBuildingConfigExportsUnitsCorrectly(self): def testExportBuildingConfigExportsLinksCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration(self.export_filepath) ) exported_links = exported_building_config.get('test_virtual_guid').get( 'links' @@ -245,7 +250,7 @@ def testExportBuildingConfigRaisesValueErrorForBadMultistate(self): export_helper = BuildingConfigExport(model) with self.assertRaises(ValueError): - export_helper.ExportInitBuildingConfiguration(self.export_filepath) + export_helper.ExportBuildingConfiguration(self.export_filepath, []) if __name__ == '__main__':