Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ aws_assume_role_lib~=1.1
boto3~=1.42
jsonschema~=4.23
odin~=2.0
requests~=2.32
pycountry~=26.2
rac_schema_validator~=1.0
shortuuid~=1.0
20 changes: 5 additions & 15 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,14 @@ attrs==26.1.0
# referencing
aws-assume-role-lib==1.7.0
# via -r requirements.in
boto3==1.43.106
boto3==1.43.109
# via
# -r requirements.in
# aws-assume-role-lib
botocore==1.43.106
botocore==1.43.109
# via
# boto3
# s3transfer
certifi==2026.7.22
# via requests
charset-normalizer==3.5.2
# via requests
idna==3.20
# via requests
jmespath==1.1.0
# via
# boto3
Expand All @@ -34,15 +28,13 @@ pycountry==26.2.16
# via -r requirements.in
python-dateutil==2.9.0.post0
# via botocore
rac-schema-validator==1.1.5
rac-schema-validator==1.1.6
# via -r requirements.in
referencing==0.37.0
# via
# jsonschema
# jsonschema-specifications
requests==2.34.2
# via -r requirements.in
rpds-py==2026.6.3
rpds-py==2026.9.1
# via
# jsonschema
# referencing
Expand All @@ -55,6 +47,4 @@ six==1.17.0
typing-extensions==4.16.0
# via referencing
urllib3==2.8.0
# via
# botocore
# requests
# via botocore
25 changes: 7 additions & 18 deletions src/mappings.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

import odin
import pycountry
import requests
import shortuuid

from .resources.configs import NOTE_TYPE_CHOICES, NOTE_TYPE_CHOICES_TRANSFORM
Expand Down Expand Up @@ -70,28 +69,18 @@ def convert_dates(value):
)


def has_online_asset(identifier, config):
if not config.get('ASSET_BASEURL'):
return False
try:
req = requests.head(f"{config['ASSET_BASEURL'].rstrip('/')}/pdfs/{identifier}")
return req.status_code == 200
except Exception as e:
logging.info(e)
return False


def has_online_instance(instances, uri, config):
def has_online_instance(instances):
"""Checks to see if there are digital objects."""
try:
digital_instances = [
v for v in instances if v.instance_type == "digital_object"]
published_instances = [v for v in digital_instances if v.digital_object.publish is True]
except AttributeError:
digital_instances = [
v for v in instances if v["instance_type"] == "digital_object"]
if len(digital_instances):
if has_online_asset(identifier_from_uri(uri), config):
return True
published_instances = [v for v in digital_instances if v["digital_object"]["publish"] is True]
if len(published_instances):
return True
return False


Expand Down Expand Up @@ -585,7 +574,7 @@ def formats(self, value):

@odin.map_field(from_field="instances", to_field="online")
def online(self, value):
return has_online_instance(value, self.source.uri, self.context)
return has_online_instance(value)

@odin.map_field(from_field="group", to_field="group")
def group(self, value):
Expand Down Expand Up @@ -655,7 +644,7 @@ def formats(self, value):

@odin.map_field(from_field="instances", to_field="online")
def online(self, value):
return has_online_instance(value, self.source.uri, self.context)
return has_online_instance(value)

@odin.map_field(from_field="instances", to_field="files", to_list=True)
def files(self, value):
Expand Down
12 changes: 0 additions & 12 deletions src/transformer.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,6 @@ def get_mapping_classes(self, object_type):
}
return type_map[object_type]

def get_online_pending(self, instances, online):
published_digital_instances = [
v for v in instances
if v.get("instance_type") == "digital_object"
and v.get("digital_object", {}).get("_resolved", {}).get("publish")
]
return bool(len(published_digital_instances) and not online)

def get_transformed_object(self, data, from_resource, mapping):
from_obj = json_codec.loads(json.dumps(data), resource=from_resource)
transformed = mapping.apply(from_obj, context=self.config)
Expand Down Expand Up @@ -204,10 +196,6 @@ def run(self, object_type, data):
transformed = self.get_transformed_object(
data, from_resource, mapping)
logging.debug(f"Object {data['uri']} transformed")
transformed['online_pending'] = self.get_online_pending(
data.get("instances", []),
transformed.get("online", False))
logging.debug(f"Online pending set for object {data['uri']}")
self.validate_transformed(transformed, schema_name)
logging.debug(f"Transformed object {data['uri']} validated")
self.send_success_message(transformed, object_type)
Expand Down
41 changes: 0 additions & 41 deletions tests/fixtures/transformer/online_instance/multiple_instances.json

This file was deleted.

This file was deleted.

18 changes: 0 additions & 18 deletions tests/fixtures/transformer/online_instance/online_instance.json

This file was deleted.

This file was deleted.

4 changes: 2 additions & 2 deletions tests/test_mapping_classes.py
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,7 @@ def test_online(self, mock_online_instance):
self.mapping.source = RecursiveNamespace.from_obj({"uri": "uri"})
self.mapping.context = None
self.assertFalse(self.mapping.online("value"))
mock_online_instance.assert_called_once_with("value", "uri", None)
mock_online_instance.assert_called_once_with("value")

@patch('src.mappings.transform_group')
def test_group(self, mock_group):
Expand Down Expand Up @@ -667,7 +667,7 @@ def test_online(self, mock_online_instance):
self.mapping.source = RecursiveNamespace.from_obj({"uri": "uri"})
self.mapping.context = None
self.assertFalse(self.mapping.online("value"))
mock_online_instance.assert_called_once_with("value", "uri", None)
mock_online_instance.assert_called_once_with("value")

@patch('src.mappings.generate_download_identifier')
@patch('src.mappings.generate_manifest_identifier')
Expand Down
47 changes: 12 additions & 35 deletions tests/test_mapping_helpers.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
from unittest.mock import patch

from src.mappings import (convert_dates, generate_download_identifier,
generate_manifest_identifier, has_online_asset,
has_online_instance, identifier_from_uri,
language_name, strip_tags, transform_formats,
transform_group, transform_language)
generate_manifest_identifier, has_online_instance,
identifier_from_uri, language_name, strip_tags,
transform_formats, transform_group,
transform_language)
from tests.test_helpers import RecursiveNamespace


Expand Down Expand Up @@ -48,41 +48,18 @@ def test_strip_tags_xml_and_regex():
assert strip_tags("a <b>broken") == "a <b>broken"


@patch("requests.head")
def test_has_online_asset(mock_head):
mock_head.return_value = RecursiveNamespace.from_obj({"status_code": 200})
assert has_online_asset(
"abc", {"ASSET_BASEURL": "https://assets.example.org"}
) is True
mock_head.assert_called_once_with("https://assets.example.org/pdfs/abc")

mock_head.reset_mock()
mock_head.return_value = RecursiveNamespace.from_obj({"status_code": 404})
assert has_online_asset("abc", {}) is False
mock_head.assert_not_called()

mock_head.reset_mock()
mock_head.return_value = RecursiveNamespace.from_obj({"status_code": 404})
assert has_online_asset("abc", {"ASSET_BASEURL": ""}) is False
mock_head.assert_not_called()


@patch("src.mappings.has_online_asset")
def test_has_online_instance(mock_online_asset):
mock_online_asset.return_value = True
def test_has_online_instance():
instances = [
{"instance_type": "digital_object"},
{"instance_type": "digital_object", "digital_object": {"publish": True}},
{"instance_type": "text"},
]
uri = "/repositories/2/resources/123"
assert has_online_instance(instances) is True

assert has_online_instance(instances, uri, {}) is True
mock_online_asset.assert_called_once_with(identifier_from_uri(uri), {})

mock_online_asset.reset_mock()
mock_online_asset.return_value = False
assert has_online_instance(instances, uri, {}) is False
mock_online_asset.assert_called_once_with(identifier_from_uri(uri), {})
instances = [
{"instance_type": "digital_object", "digital_object": {"publish": False}},
{"instance_type": "text"},
]
assert has_online_instance(instances) is False


@patch("src.mappings.odin.codecs.json_codec.loads")
Expand Down
8 changes: 4 additions & 4 deletions tests/test_requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
boto3==1.43.106
boto3==1.43.109
# via moto
botocore==1.43.106
botocore==1.43.109
# via
# boto3
# moto
Expand All @@ -17,13 +17,13 @@ cryptography==50.0.2
# via moto
idna==3.20
# via requests
iniconfig==2.3.0
iniconfig==2.3.1
# via pytest
jmespath==1.1.0
# via
# boto3
# botocore
markupsafe==3.0.3
markupsafe==3.0.4
# via werkzeug
moto==5.2.3
# via -r tests/test_requirements.in
Expand Down
Loading
Loading