diff --git a/src/mappings.py b/src/mappings.py index 0e07255..6c00b84 100644 --- a/src/mappings.py +++ b/src/mappings.py @@ -1,6 +1,8 @@ import json +import logging import re import xml.etree.ElementTree as ET +from os import getenv import odin import pycountry @@ -19,6 +21,9 @@ SourceRef, SourceResource, SourceStructuredDate, SourceSubject) +logger = logging.getLogger() +logger.setLevel(getenv('LOGGING_LEVEL', logging.INFO)) + def identifier_from_uri(uri): """Creates a short UUID. @@ -68,8 +73,12 @@ def convert_dates(value): def has_online_asset(identifier, config): if not config.get('ASSET_BASEURL'): return False - req = requests.head(f"{config['ASSET_BASEURL'].rstrip('/')}/pdfs/{identifier}") - return req.status_code == 200 + try: + req = requests.head(f"{config['ASSET_BASEURL'].rstrip('/')}/pdfs/{identifier}") + return req.status_code == 200 + except Exception as e: + logging.info(e) + return False def has_online_instance(instances, uri, config): @@ -653,21 +662,20 @@ def files(self, value): files = [] for instance in value: if instance.digital_object and instance.digital_object.publish: - files.append( - FileObject( - title=instance.digital_object.title, - download=generate_download_identifier( - self.source.to_dict(), - instance.digital_object.to_dict(), - self.context - ), - manifest=generate_manifest_identifier( - self.source.to_dict(), - instance.digital_object.to_dict(), - self.context - ) + file_object = FileObject(title=instance.digital_object.title) + if any([f.use_statement == 'download' for f in instance.digital_object.file_versions]): + file_object.download = generate_download_identifier( + self.source.to_dict(), + instance.digital_object.to_dict(), + self.context ) - ) + if any([f.use_statement == 'iiif-manifest' for f in instance.digital_object.file_versions]): + file_object.manifest = generate_manifest_identifier( + self.source.to_dict(), + instance.digital_object.to_dict(), + self.context) + if any([file_object.download, file_object.manifest]): + files.append(file_object) return files @odin.map_field(from_field="group", to_field="group") diff --git a/src/resources/configs.py b/src/resources/configs.py index 1e1eeba..4c9c312 100644 --- a/src/resources/configs.py +++ b/src/resources/configs.py @@ -509,3 +509,9 @@ ('temporal', 'Temporal'), ('topical', 'Topical'), ) + +FILE_VERSION_USE_STATEMENT_CHOICES = ( + ('aip', 'aip'), + ('download', 'download'), + ('iiif-manifest', 'iiif-manifest'), +) diff --git a/src/resources/source.py b/src/resources/source.py index 3a9f1cb..96b0554 100644 --- a/src/resources/source.py +++ b/src/resources/source.py @@ -65,10 +65,17 @@ class SourceStructuredDate(odin.Resource): structured_date_range = odin.DictAs(SourceStructuredDateRange, null=True) +class SourceDigitalObjectFileVersion(odin.Resource): + """A file version for digital objects.""" + use_statement = odin.StringField(choices=configs.FILE_VERSION_USE_STATEMENT_CHOICES) + + class SourceDigitalObject(odin.Resource): + """A digital representation of an archival object.""" title = odin.StringField() digital_object_id = odin.StringField() publish = odin.BooleanField() + file_versions = odin.ListOf(SourceDigitalObjectFileVersion) class SourceExtent(odin.Resource): diff --git a/src/transformer.py b/src/transformer.py index 7e67789..9664d0c 100644 --- a/src/transformer.py +++ b/src/transformer.py @@ -18,6 +18,9 @@ SourceAgentPerson, SourceArchivalObject, SourceResource, SourceSubject) +logger = logging.getLogger() +logger.setLevel(getenv('LOGGING_LEVEL', logging.INFO)) + def get_config(environment, aws_region, service_name): """Fetch config values from SSM Parameter Store by path.""" @@ -103,7 +106,7 @@ def remove_keys_from_dict(self, data, target_key="$"): modified_dict = {} if hasattr(data, "items"): for key, value in data.items(): - if key != target_key: + if (value is not None and key != target_key): if isinstance(value, dict): modified_dict[key] = self.remove_keys_from_dict( value, target_key=target_key) @@ -194,17 +197,23 @@ def send_error_message(self, exception, object_type, object_id): }) def run(self, object_type, data): + logging.debug(f"Running transform for {object_type} {data['uri']}") try: from_resource, mapping, schema_name = self.get_mapping_classes( object_type) transformed = self.get_transformed_object( data, from_resource, mapping) + logging.debug(f"Object {data['uri']} transformed") transformed['online_pending'] = self.get_online_pending( data.get("instances", []), transformed.get("online", False)) + logging.debug(f"Online pending set for object {data['uri']}") self.validate_transformed(transformed, schema_name) + logging.debug(f"Transformed object {data['uri']} validated") self.send_success_message(transformed, object_type) + logging.info(f"Transformation for {object_type} {data['uri']} complete") except Exception as e: + logging.error(e) self.send_error_message( e, object_type, identifier_from_uri( data['uri'])) @@ -212,6 +221,7 @@ def run(self, object_type, data): def lambda_handler(event, context): """Process SQS batch.""" + logger.info("Message batch received.") transformer = Transformer() diff --git a/tests/fixtures/transformer/archival_object/2329.json b/tests/fixtures/transformer/archival_object/2329.json index 2f7d67d..c1a77fb 100644 --- a/tests/fixtures/transformer/archival_object/2329.json +++ b/tests/fixtures/transformer/archival_object/2329.json @@ -259,7 +259,7 @@ "lock_version": 0, "publish": true, "system_mtime": "2022-05-14T15:33:00Z", - "use_statement": "master", + "use_statement": "download", "user_mtime": "2022-05-14T15:33:00Z" } ], diff --git a/tests/test_mapping_classes.py b/tests/test_mapping_classes.py index fcc07fc..849890b 100644 --- a/tests/test_mapping_classes.py +++ b/tests/test_mapping_classes.py @@ -672,27 +672,65 @@ def test_online(self, mock_online_instance): @patch('src.mappings.generate_download_identifier') @patch('src.mappings.generate_manifest_identifier') def test_files(self, mock_manifest_id, mock_download_id): + # TODO add tests for file versions mock_manifest_id.return_value = "manifest_id" mock_download_id.return_value = "download_id" self.mapping.source = RecursiveNamespace.from_obj({}) self.mapping.context = "https://context.org" output = self.mapping.files([ - RecursiveNamespace.from_obj({"digital_object": {"title": "digital object title", "publish": True}}), - RecursiveNamespace.from_obj({"digital_object": {"title": "digital object title", "publish": False}}), - RecursiveNamespace.from_obj({"digital_object": None})]) + RecursiveNamespace.from_obj({ + "digital_object": { + "title": "digital object title", + "publish": True, + "file_versions": [ + RecursiveNamespace.from_obj({"use_statement": "download"}), + RecursiveNamespace.from_obj({"use_statement": "iiif-manifest"}) + ]}}), + RecursiveNamespace.from_obj({ + "digital_object": { + "title": "digital object title", + "publish": True, + "file_versions": [ + RecursiveNamespace.from_obj({"use_statement": "download"})]}}), + RecursiveNamespace.from_obj({ + "digital_object": { + "title": "digital object title", + "publish": True, + "file_versions": [ + RecursiveNamespace.from_obj({"use_statement": "aip"})]}}), + RecursiveNamespace.from_obj({ + "digital_object": { + "title": "digital object title", + "publish": False, + "file_versions": []}}), + RecursiveNamespace.from_obj({"digital_object": None}) + ]) self.assertEqual( json.loads(json_codec.dumps(output)), - [{ - 'title': 'digital object title', - 'download': 'download_id', - 'manifest': 'manifest_id', - '$': 'src.resources.rac.FileObject' - }]) - - mock_download_id.assert_called_once_with( - {}, {"title": "digital object title", "publish": True}, "https://context.org") + [ + { + "$": "src.resources.rac.FileObject", + "title": "digital object title", + "download": "download_id", + "manifest": "manifest_id" + }, + { + "$": "src.resources.rac.FileObject", + "title": "digital object title", + "download": "download_id", + "manifest": None + } + ]) + + mock_download_id.assert_has_calls([ + call({}, {'title': 'digital object title', 'publish': True, 'file_versions': [ + {'use_statement': 'download'}, {'use_statement': 'iiif-manifest'}]}, 'https://context.org'), + call({}, {'title': 'digital object title', 'publish': True, 'file_versions': [{'use_statement': 'download'}]}, 'https://context.org') + ]) mock_manifest_id.assert_called_once_with( - {}, {"title": "digital object title", "publish": True}, "https://context.org") + {}, {'title': 'digital object title', 'publish': True, 'file_versions': [ + {'use_statement': 'download'}, {'use_statement': 'iiif-manifest'}]}, 'https://context.org' + ) @patch('src.mappings.transform_group') def test_group(self, mock_group): diff --git a/tests/test_transformer.py b/tests/test_transformer.py index fc9927c..4a8bc54 100644 --- a/tests/test_transformer.py +++ b/tests/test_transformer.py @@ -92,11 +92,11 @@ def test_run( mock_online_pending.return_value = False mock_validate.return_value = True - self.transformer.run('agent_person', {}) + self.transformer.run('agent_person', {"uri": "54321"}) mock_mapping.assert_called_once_with('agent_person') mock_get_transformed.assert_called_once_with( - {}, SourceAgentPerson, SourceAgentPersonToAgent) + {"uri": "54321"}, SourceAgentPerson, SourceAgentPersonToAgent) mock_validate.assert_called_once_with( transformed_object, self.transformer.config['SCHEMA_AGENT']) mock_success_message.assert_called_once_with( @@ -185,6 +185,23 @@ def test_remove_keys_from_dict_removes_nested_dollar_keys(self): self.assertEqual(cleaned["lst"][0], "a") self.assertEqual(cleaned["lst"][1], {"c": 4}) + def test_remove_keys_from_dict_removes_null_values(self): + payload = { + "uri": "/x/1", + "null": None, + "nested": {"keep": 1, "items": [{"a": 1}, {"b": None}]}, + "lst": ["a", {"c": None}], + } + cleaned = self.transformer.remove_keys_from_dict( + payload, target_key="$") + self.assertEqual( + cleaned, + { + "uri": "/x/1", + "nested": {"keep": 1, "items": [{"a": 1}, {}]}, + "lst": ["a", {}], + }) + @patch('src.transformer.is_valid') def test_validate_transformed(self, mock_is_valid): schema_name = 'agent.json'