From 36ee9c837f7c528d656bf52f024008214fabdb0d Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 7 Oct 2026 13:54:29 -0400 Subject: [PATCH 1/3] remove online_pending --- src/transformer.py | 12 ------------ tests/test_transformer.py | 21 --------------------- 2 files changed, 33 deletions(-) diff --git a/src/transformer.py b/src/transformer.py index 9664d0c..f27a11e 100644 --- a/src/transformer.py +++ b/src/transformer.py @@ -88,14 +88,6 @@ def get_mapping_classes(self, object_type): } return type_map[object_type] - def get_online_pending(self, instances, online): - published_digital_instances = [ - v for v in instances - if v.get("instance_type") == "digital_object" - and v.get("digital_object", {}).get("_resolved", {}).get("publish") - ] - return bool(len(published_digital_instances) and not online) - def get_transformed_object(self, data, from_resource, mapping): from_obj = json_codec.loads(json.dumps(data), resource=from_resource) transformed = mapping.apply(from_obj, context=self.config) @@ -204,10 +196,6 @@ def run(self, object_type, data): transformed = self.get_transformed_object( data, from_resource, mapping) logging.debug(f"Object {data['uri']} transformed") - transformed['online_pending'] = self.get_online_pending( - data.get("instances", []), - transformed.get("online", False)) - logging.debug(f"Online pending set for object {data['uri']}") self.validate_transformed(transformed, schema_name) logging.debug(f"Transformed object {data['uri']} validated") self.send_success_message(transformed, object_type) diff --git a/tests/test_transformer.py b/tests/test_transformer.py index 4a8bc54..df3c594 100644 --- a/tests/test_transformer.py +++ b/tests/test_transformer.py @@ -71,7 +71,6 @@ def setUp(self, mock_config): @patch('src.transformer.Transformer.get_mapping_classes') @patch('src.transformer.Transformer.get_transformed_object') - @patch('src.transformer.Transformer.get_online_pending') @patch('src.transformer.Transformer.validate_transformed') @patch('src.transformer.Transformer.send_success_message') @patch('src.transformer.Transformer.send_error_message') @@ -80,7 +79,6 @@ def test_run( mock_error_message, mock_success_message, mock_validate, - mock_online_pending, mock_get_transformed, mock_mapping): transformed_object = {"uri": "12345"} @@ -89,7 +87,6 @@ def test_run( SourceAgentPersonToAgent, self.transformer.config['SCHEMA_AGENT']) mock_get_transformed.return_value = transformed_object - mock_online_pending.return_value = False mock_validate.return_value = True self.transformer.run('agent_person', {"uri": "54321"}) @@ -138,24 +135,6 @@ def test_get_mapping_classes(self): with self.assertRaises(KeyError): self.transformer.get_mapping_classes("not_a_real_type") - def test_get_online_pending(self): - online_dir = self.fixtures_dir / "online_instance" - if not online_dir.exists(): - self.skipTest(f"Missing fixture directory: {online_dir}") - - cases = [ - ("no_online_instances.json", False, False), - ("online_instance.json", False, True), - ("no_online_instances.json", True, False), - ("online_instance.json", True, False), - ("unpublished_online_instance.json", False, False), - ] - for fixture, online, expected in cases: - fixture_path = online_dir / fixture - instances = load_fixture(fixture_path) - output = self.transformer.get_online_pending(instances, online) - self.assertEqual(output, expected) - @patch('src.mappings.SourceAgentPersonToAgent.apply') @patch('src.transformer.Transformer.remove_keys_from_dict') def test_get_transformed_object(self, mock_remove_keys, mock_apply): From 824379fb2cf8489d39ef7307872532c47ef1c08d Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 7 Oct 2026 15:43:51 -0400 Subject: [PATCH 2/3] remove check for downloadable file --- requirements.in | 1 - requirements.txt | 20 ++++----------- src/mappings.py | 25 ++++++------------- tests/test_mapping_classes.py | 4 +-- tests/test_mapping_helpers.py | 47 +++++++++-------------------------- tests/test_requirements.txt | 8 +++--- 6 files changed, 30 insertions(+), 75 deletions(-) diff --git a/requirements.in b/requirements.in index 291f8b7..e641e04 100644 --- a/requirements.in +++ b/requirements.in @@ -2,7 +2,6 @@ aws_assume_role_lib~=1.1 boto3~=1.42 jsonschema~=4.23 odin~=2.0 -requests~=2.32 pycountry~=26.2 rac_schema_validator~=1.0 shortuuid~=1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f69c6be..7f4e3ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,20 +4,14 @@ attrs==26.1.0 # referencing aws-assume-role-lib==1.7.0 # via -r requirements.in -boto3==1.43.106 +boto3==1.43.109 # via # -r requirements.in # aws-assume-role-lib -botocore==1.43.106 +botocore==1.43.109 # via # boto3 # s3transfer -certifi==2026.7.22 - # via requests -charset-normalizer==3.5.2 - # via requests -idna==3.20 - # via requests jmespath==1.1.0 # via # boto3 @@ -34,15 +28,13 @@ pycountry==26.2.16 # via -r requirements.in python-dateutil==2.9.0.post0 # via botocore -rac-schema-validator==1.1.5 +rac-schema-validator==1.1.6 # via -r requirements.in referencing==0.37.0 # via # jsonschema # jsonschema-specifications -requests==2.34.2 - # via -r requirements.in -rpds-py==2026.6.3 +rpds-py==2026.9.1 # via # jsonschema # referencing @@ -55,6 +47,4 @@ six==1.17.0 typing-extensions==4.16.0 # via referencing urllib3==2.8.0 - # via - # botocore - # requests + # via botocore diff --git a/src/mappings.py b/src/mappings.py index 6c00b84..14346dc 100644 --- a/src/mappings.py +++ b/src/mappings.py @@ -6,7 +6,6 @@ import odin import pycountry -import requests import shortuuid from .resources.configs import NOTE_TYPE_CHOICES, NOTE_TYPE_CHOICES_TRANSFORM @@ -70,28 +69,18 @@ def convert_dates(value): ) -def has_online_asset(identifier, config): - if not config.get('ASSET_BASEURL'): - return False - try: - req = requests.head(f"{config['ASSET_BASEURL'].rstrip('/')}/pdfs/{identifier}") - return req.status_code == 200 - except Exception as e: - logging.info(e) - return False - - -def has_online_instance(instances, uri, config): +def has_online_instance(instances): """Checks to see if there are digital objects.""" try: digital_instances = [ v for v in instances if v.instance_type == "digital_object"] + published_instances = [v for v in digital_instances if v.digital_object.publish is True] except AttributeError: digital_instances = [ v for v in instances if v["instance_type"] == "digital_object"] - if len(digital_instances): - if has_online_asset(identifier_from_uri(uri), config): - return True + published_instances = [v for v in digital_instances if v["digital_object"]["publish"] is True] + if len(published_instances): + return True return False @@ -585,7 +574,7 @@ def formats(self, value): @odin.map_field(from_field="instances", to_field="online") def online(self, value): - return has_online_instance(value, self.source.uri, self.context) + return has_online_instance(value) @odin.map_field(from_field="group", to_field="group") def group(self, value): @@ -655,7 +644,7 @@ def formats(self, value): @odin.map_field(from_field="instances", to_field="online") def online(self, value): - return has_online_instance(value, self.source.uri, self.context) + return has_online_instance(value) @odin.map_field(from_field="instances", to_field="files", to_list=True) def files(self, value): diff --git a/tests/test_mapping_classes.py b/tests/test_mapping_classes.py index 849890b..d4be719 100644 --- a/tests/test_mapping_classes.py +++ b/tests/test_mapping_classes.py @@ -558,7 +558,7 @@ def test_online(self, mock_online_instance): self.mapping.source = RecursiveNamespace.from_obj({"uri": "uri"}) self.mapping.context = None self.assertFalse(self.mapping.online("value")) - mock_online_instance.assert_called_once_with("value", "uri", None) + mock_online_instance.assert_called_once_with("value") @patch('src.mappings.transform_group') def test_group(self, mock_group): @@ -667,7 +667,7 @@ def test_online(self, mock_online_instance): self.mapping.source = RecursiveNamespace.from_obj({"uri": "uri"}) self.mapping.context = None self.assertFalse(self.mapping.online("value")) - mock_online_instance.assert_called_once_with("value", "uri", None) + mock_online_instance.assert_called_once_with("value") @patch('src.mappings.generate_download_identifier') @patch('src.mappings.generate_manifest_identifier') diff --git a/tests/test_mapping_helpers.py b/tests/test_mapping_helpers.py index 7ebd0c1..95ab881 100644 --- a/tests/test_mapping_helpers.py +++ b/tests/test_mapping_helpers.py @@ -1,10 +1,10 @@ from unittest.mock import patch from src.mappings import (convert_dates, generate_download_identifier, - generate_manifest_identifier, has_online_asset, - has_online_instance, identifier_from_uri, - language_name, strip_tags, transform_formats, - transform_group, transform_language) + generate_manifest_identifier, has_online_instance, + identifier_from_uri, language_name, strip_tags, + transform_formats, transform_group, + transform_language) from tests.test_helpers import RecursiveNamespace @@ -48,41 +48,18 @@ def test_strip_tags_xml_and_regex(): assert strip_tags("a broken") == "a broken" -@patch("requests.head") -def test_has_online_asset(mock_head): - mock_head.return_value = RecursiveNamespace.from_obj({"status_code": 200}) - assert has_online_asset( - "abc", {"ASSET_BASEURL": "https://assets.example.org"} - ) is True - mock_head.assert_called_once_with("https://assets.example.org/pdfs/abc") - - mock_head.reset_mock() - mock_head.return_value = RecursiveNamespace.from_obj({"status_code": 404}) - assert has_online_asset("abc", {}) is False - mock_head.assert_not_called() - - mock_head.reset_mock() - mock_head.return_value = RecursiveNamespace.from_obj({"status_code": 404}) - assert has_online_asset("abc", {"ASSET_BASEURL": ""}) is False - mock_head.assert_not_called() - - -@patch("src.mappings.has_online_asset") -def test_has_online_instance(mock_online_asset): - mock_online_asset.return_value = True +def test_has_online_instance(): instances = [ - {"instance_type": "digital_object"}, + {"instance_type": "digital_object", "digital_object": {"publish": True}}, {"instance_type": "text"}, ] - uri = "/repositories/2/resources/123" + assert has_online_instance(instances) is True - assert has_online_instance(instances, uri, {}) is True - mock_online_asset.assert_called_once_with(identifier_from_uri(uri), {}) - - mock_online_asset.reset_mock() - mock_online_asset.return_value = False - assert has_online_instance(instances, uri, {}) is False - mock_online_asset.assert_called_once_with(identifier_from_uri(uri), {}) + instances = [ + {"instance_type": "digital_object", "digital_object": {"publish": False}}, + {"instance_type": "text"}, + ] + assert has_online_instance(instances) is False @patch("src.mappings.odin.codecs.json_codec.loads") diff --git a/tests/test_requirements.txt b/tests/test_requirements.txt index e6a889e..56ffe04 100644 --- a/tests/test_requirements.txt +++ b/tests/test_requirements.txt @@ -1,6 +1,6 @@ -boto3==1.43.106 +boto3==1.43.109 # via moto -botocore==1.43.106 +botocore==1.43.109 # via # boto3 # moto @@ -17,13 +17,13 @@ cryptography==50.0.2 # via moto idna==3.20 # via requests -iniconfig==2.3.0 +iniconfig==2.3.1 # via pytest jmespath==1.1.0 # via # boto3 # botocore -markupsafe==3.0.3 +markupsafe==3.0.4 # via werkzeug moto==5.2.3 # via -r tests/test_requirements.in From 3b1bee3c1e00e7a5a4b16f65015248292e562ac0 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 7 Oct 2026 15:49:45 -0400 Subject: [PATCH 3/3] remove unused fixtures --- .../online_instance/multiple_instances.json | 41 ------------------- .../online_instance/no_online_instances.json | 25 ----------- .../online_instance/online_instance.json | 18 -------- .../unpublished_online_instance.json | 18 -------- 4 files changed, 102 deletions(-) delete mode 100644 tests/fixtures/transformer/online_instance/multiple_instances.json delete mode 100644 tests/fixtures/transformer/online_instance/no_online_instances.json delete mode 100644 tests/fixtures/transformer/online_instance/online_instance.json delete mode 100644 tests/fixtures/transformer/online_instance/unpublished_online_instance.json diff --git a/tests/fixtures/transformer/online_instance/multiple_instances.json b/tests/fixtures/transformer/online_instance/multiple_instances.json deleted file mode 100644 index 397e6c7..0000000 --- a/tests/fixtures/transformer/online_instance/multiple_instances.json +++ /dev/null @@ -1,41 +0,0 @@ -[ - { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "instance_type": "digital_object", - "is_representative": false, - "jsonmodel_type": "instance", - "last_modified_by": "admin", - "lock_version": 0, - "system_mtime": "2020-09-21T17:01:58Z", - "user_mtime": "2020-09-21T17:01:58Z", - "digital_object": { - "_resolved": { - "publish": true - } - } - }, - { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "instance_type": "mixed materials", - "is_representative": false, - "jsonmodel_type": "instance", - "last_modified_by": "admin", - "lock_version": 0, - "sub_container": { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "jsonmodel_type": "sub_container", - "last_modified_by": "admin", - "lock_version": 0, - "system_mtime": "2020-09-21T17:01:58Z", - "top_container": { - "ref": "/repositories/101/top_containers/99" - }, - "user_mtime": "2020-09-21T17:01:58Z" - }, - "system_mtime": "2020-09-21T17:01:58Z", - "user_mtime": "2020-09-21T17:01:58Z" - } -] diff --git a/tests/fixtures/transformer/online_instance/no_online_instances.json b/tests/fixtures/transformer/online_instance/no_online_instances.json deleted file mode 100644 index fe0665e..0000000 --- a/tests/fixtures/transformer/online_instance/no_online_instances.json +++ /dev/null @@ -1,25 +0,0 @@ -[ - { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "instance_type": "mixed materials", - "is_representative": false, - "jsonmodel_type": "instance", - "last_modified_by": "admin", - "lock_version": 0, - "sub_container": { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "jsonmodel_type": "sub_container", - "last_modified_by": "admin", - "lock_version": 0, - "system_mtime": "2020-09-21T17:01:58Z", - "top_container": { - "ref": "/repositories/101/top_containers/99" - }, - "user_mtime": "2020-09-21T17:01:58Z" - }, - "system_mtime": "2020-09-21T17:01:58Z", - "user_mtime": "2020-09-21T17:01:58Z" - } -] diff --git a/tests/fixtures/transformer/online_instance/online_instance.json b/tests/fixtures/transformer/online_instance/online_instance.json deleted file mode 100644 index 9410e26..0000000 --- a/tests/fixtures/transformer/online_instance/online_instance.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "instance_type": "digital_object", - "is_representative": false, - "jsonmodel_type": "instance", - "last_modified_by": "admin", - "lock_version": 0, - "system_mtime": "2020-09-21T17:01:58Z", - "user_mtime": "2020-09-21T17:01:58Z", - "digital_object": { - "_resolved": { - "publish": true - } - } - } -] diff --git a/tests/fixtures/transformer/online_instance/unpublished_online_instance.json b/tests/fixtures/transformer/online_instance/unpublished_online_instance.json deleted file mode 100644 index a034545..0000000 --- a/tests/fixtures/transformer/online_instance/unpublished_online_instance.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "create_time": "2020-09-21T17:01:58Z", - "created_by": "admin", - "instance_type": "digital_object", - "is_representative": false, - "jsonmodel_type": "instance", - "last_modified_by": "admin", - "lock_version": 0, - "system_mtime": "2020-09-21T17:01:58Z", - "user_mtime": "2020-09-21T17:01:58Z", - "digital_object": { - "_resolved": { - "publish": false - } - } - } -]