fix: Support digital objects with no IIIF manifest

src/mappings.py

@@ -1,6 +1,8 @@
 import json
+import logging
 import re
 import xml.etree.ElementTree as ET
+from os import getenv
 
 import odin
 import pycountry
@@ -19,6 +21,9 @@
                                SourceRef, SourceResource, SourceStructuredDate,
                                SourceSubject)
 
+logger = logging.getLogger()
+logger.setLevel(getenv('LOGGING_LEVEL', logging.INFO))
+
 
 def identifier_from_uri(uri):
     """Creates a short UUID.
@@ -68,8 +73,12 @@ def convert_dates(value):
 def has_online_asset(identifier, config):
     if not config.get('ASSET_BASEURL'):
         return False
-    req = requests.head(f"{config['ASSET_BASEURL'].rstrip('/')}/pdfs/{identifier}")
-    return req.status_code == 200
+    try:
+        req = requests.head(f"{config['ASSET_BASEURL'].rstrip('/')}/pdfs/{identifier}")
+        return req.status_code == 200
+    except Exception as e:
+        logging.info(e)
+        return False
 
 
 def has_online_instance(instances, uri, config):
@@ -653,21 +662,20 @@ def files(self, value):
         files = []
         for instance in value:
             if instance.digital_object and instance.digital_object.publish:
-                files.append(
-                    FileObject(
-                        title=instance.digital_object.title,
-                        download=generate_download_identifier(
-                            self.source.to_dict(),
-                            instance.digital_object.to_dict(),
-                            self.context
-                        ),
-                        manifest=generate_manifest_identifier(
-                            self.source.to_dict(),
-                            instance.digital_object.to_dict(),
-                            self.context
-                        )
+                file_object = FileObject(title=instance.digital_object.title)
+                if any([f.use_statement == 'download' for f in instance.digital_object.file_versions]):
+                    file_object.download = generate_download_identifier(
+                        self.source.to_dict(),
+                        instance.digital_object.to_dict(),
+                        self.context
                     )
-                )
+                if any([f.use_statement == 'iiif-manifest' for f in instance.digital_object.file_versions]):
+                    file_object.manifest = generate_manifest_identifier(
+                        self.source.to_dict(),
+                        instance.digital_object.to_dict(),
+                        self.context)
+                if any([file_object.download, file_object.manifest]):
+                    files.append(file_object)
         return files
 
     @odin.map_field(from_field="group", to_field="group")

src/resources/configs.py

@@ -509,3 +509,9 @@
     ('temporal', 'Temporal'),
     ('topical', 'Topical'),
 )
+
+FILE_VERSION_USE_STATEMENT_CHOICES = (
+    ('aip', 'aip'),
+    ('download', 'download'),
+    ('iiif-manifest', 'iiif-manifest'),
+)

src/resources/source.py

@@ -65,10 +65,17 @@ class SourceStructuredDate(odin.Resource):
     structured_date_range = odin.DictAs(SourceStructuredDateRange, null=True)
 
 
+class SourceDigitalObjectFileVersion(odin.Resource):
+    """A file version for digital objects."""
+    use_statement = odin.StringField(choices=configs.FILE_VERSION_USE_STATEMENT_CHOICES)
+
+
 class SourceDigitalObject(odin.Resource):
+    """A digital representation of an archival object."""
     title = odin.StringField()
     digital_object_id = odin.StringField()
     publish = odin.BooleanField()
+    file_versions = odin.ListOf(SourceDigitalObjectFileVersion)
 
 
 class SourceExtent(odin.Resource):

src/transformer.py

@@ -18,6 +18,9 @@
                                SourceAgentPerson, SourceArchivalObject,
                                SourceResource, SourceSubject)
 
+logger = logging.getLogger()
+logger.setLevel(getenv('LOGGING_LEVEL', logging.INFO))
+
 
 def get_config(environment, aws_region, service_name):
     """Fetch config values from SSM Parameter Store by path."""
@@ -103,7 +106,7 @@ def remove_keys_from_dict(self, data, target_key="$"):
         modified_dict = {}
         if hasattr(data, "items"):
             for key, value in data.items():
-                if key != target_key:
+                if (value is not None and key != target_key):
                     if isinstance(value, dict):
                         modified_dict[key] = self.remove_keys_from_dict(
                             value, target_key=target_key)
@@ -194,24 +197,31 @@ def send_error_message(self, exception, object_type, object_id):
             })
 
     def run(self, object_type, data):
+        logging.debug(f"Running transform for {object_type} {data['uri']}")
         try:
             from_resource, mapping, schema_name = self.get_mapping_classes(
                 object_type)
             transformed = self.get_transformed_object(
                 data, from_resource, mapping)
+            logging.debug(f"Object {data['uri']} transformed")
             transformed['online_pending'] = self.get_online_pending(
                 data.get("instances", []),
                 transformed.get("online", False))
+            logging.debug(f"Online pending set for object {data['uri']}")
             self.validate_transformed(transformed, schema_name)
+            logging.debug(f"Transformed object {data['uri']} validated")
             self.send_success_message(transformed, object_type)
+            logging.info(f"Transformation for {object_type} {data['uri']} complete")
         except Exception as e:
+            logging.error(e)
             self.send_error_message(
                 e, object_type, identifier_from_uri(
                     data['uri']))
 
 
 def lambda_handler(event, context):
     """Process SQS batch."""
+    logger.info("Message batch received.")
 
     transformer = Transformer()
 

tests/fixtures/transformer/archival_object/2329.json

@@ -259,7 +259,7 @@
                         "lock_version": 0,
                         "publish": true,
                         "system_mtime": "2022-05-14T15:33:00Z",
-                        "use_statement": "master",
+                        "use_statement": "download",
                         "user_mtime": "2022-05-14T15:33:00Z"
                     }
                 ],

tests/test_mapping_classes.py

@@ -672,27 +672,65 @@ def test_online(self, mock_online_instance):
     @patch('src.mappings.generate_download_identifier')
     @patch('src.mappings.generate_manifest_identifier')
     def test_files(self, mock_manifest_id, mock_download_id):
+        # TODO add tests for file versions
         mock_manifest_id.return_value = "manifest_id"
         mock_download_id.return_value = "download_id"
         self.mapping.source = RecursiveNamespace.from_obj({})
         self.mapping.context = "https://context.org"
         output = self.mapping.files([
-            RecursiveNamespace.from_obj({"digital_object": {"title": "digital object title", "publish": True}}),
-            RecursiveNamespace.from_obj({"digital_object": {"title": "digital object title", "publish": False}}),
-            RecursiveNamespace.from_obj({"digital_object": None})])
+            RecursiveNamespace.from_obj({
+                "digital_object": {
+                    "title": "digital object title",
+                    "publish": True,
+                    "file_versions": [
+                        RecursiveNamespace.from_obj({"use_statement": "download"}),
+                        RecursiveNamespace.from_obj({"use_statement": "iiif-manifest"})
+                    ]}}),
+            RecursiveNamespace.from_obj({
+                "digital_object": {
+                    "title": "digital object title",
+                    "publish": True,
+                    "file_versions": [
+                        RecursiveNamespace.from_obj({"use_statement": "download"})]}}),
+            RecursiveNamespace.from_obj({
+                "digital_object": {
+                    "title": "digital object title",
+                    "publish": True,
+                    "file_versions": [
+                        RecursiveNamespace.from_obj({"use_statement": "aip"})]}}),
+            RecursiveNamespace.from_obj({
+                "digital_object": {
+                    "title": "digital object title",
+                    "publish": False,
+                    "file_versions": []}}),
+            RecursiveNamespace.from_obj({"digital_object": None})
+        ])
         self.assertEqual(
             json.loads(json_codec.dumps(output)),
-            [{
-                'title': 'digital object title',
-                'download': 'download_id',
-                'manifest': 'manifest_id',
-                '$': 'src.resources.rac.FileObject'
-            }])
-
-        mock_download_id.assert_called_once_with(
-            {}, {"title": "digital object title", "publish": True}, "https://context.org")
+            [
+                {
+                    "$": "src.resources.rac.FileObject",
+                    "title": "digital object title",
+                    "download": "download_id",
+                    "manifest": "manifest_id"
+                },
+                {
+                    "$": "src.resources.rac.FileObject",
+                    "title": "digital object title",
+                    "download": "download_id",
+                    "manifest": None
+                }
+            ])
+
+        mock_download_id.assert_has_calls([
+            call({}, {'title': 'digital object title', 'publish': True, 'file_versions': [
+                 {'use_statement': 'download'}, {'use_statement': 'iiif-manifest'}]}, 'https://context.org'),
+            call({}, {'title': 'digital object title', 'publish': True, 'file_versions': [{'use_statement': 'download'}]}, 'https://context.org')
+        ])
         mock_manifest_id.assert_called_once_with(
-            {}, {"title": "digital object title", "publish": True}, "https://context.org")
+            {}, {'title': 'digital object title', 'publish': True, 'file_versions': [
+                {'use_statement': 'download'}, {'use_statement': 'iiif-manifest'}]}, 'https://context.org'
+        )
 
     @patch('src.mappings.transform_group')
     def test_group(self, mock_group):

tests/test_transformer.py

@@ -92,11 +92,11 @@ def test_run(
         mock_online_pending.return_value = False
         mock_validate.return_value = True
 
-        self.transformer.run('agent_person', {})
+        self.transformer.run('agent_person', {"uri": "54321"})
 
         mock_mapping.assert_called_once_with('agent_person')
         mock_get_transformed.assert_called_once_with(
-            {}, SourceAgentPerson, SourceAgentPersonToAgent)
+            {"uri": "54321"}, SourceAgentPerson, SourceAgentPersonToAgent)
         mock_validate.assert_called_once_with(
             transformed_object, self.transformer.config['SCHEMA_AGENT'])
         mock_success_message.assert_called_once_with(
@@ -185,6 +185,23 @@ def test_remove_keys_from_dict_removes_nested_dollar_keys(self):
         self.assertEqual(cleaned["lst"][0], "a")
         self.assertEqual(cleaned["lst"][1], {"c": 4})
 
+    def test_remove_keys_from_dict_removes_null_values(self):
+        payload = {
+            "uri": "/x/1",
+            "null": None,
+            "nested": {"keep": 1, "items": [{"a": 1}, {"b": None}]},
+            "lst": ["a", {"c": None}],
+        }
+        cleaned = self.transformer.remove_keys_from_dict(
+            payload, target_key="$")
+        self.assertEqual(
+            cleaned,
+            {
+                "uri": "/x/1",
+                "nested": {"keep": 1, "items": [{"a": 1}, {}]},
+                "lst": ["a", {}],
+            })
+
     @patch('src.transformer.is_valid')
     def test_validate_transformed(self, mock_is_valid):
         schema_name = 'agent.json'