diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio b/docs/adr/hermes-prov-diagram/hermes-prov.drawio new file mode 100644 index 00000000..70b06617 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio @@ -0,0 +1,4162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license b/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license new file mode 100644 index 00000000..e4d2c6e9 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.drawio.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) + +SPDX-License-Identifier: CC-BY-SA-4.0 \ No newline at end of file diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg b/docs/adr/hermes-prov-diagram/hermes-prov.svg new file mode 100644 index 00000000..56e2a032 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg @@ -0,0 +1,4 @@ + + + +
wasGeneratedBy
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasInfluencedBy
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasInfluencedBy
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
used
used
wasAssociatedWith
wasGeneratedBy
wasAssociatedWith
wasAttributedTo
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
wasInformedBy
wasGeneratedBy
used
used
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
used
wasAssociatedWith
wasInformedBy
used
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasInformedBy
used
used
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasAssociatedWith
wasInformedBy
used
wasInformedBy
wasGeneratedBy
wasInformedBy
used
wasDerivedFrom
wasGeneratedBy
used
wasInformedBy
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
used
wasGeneratedBy
wasInformedBy
used
wasGeneratedBy
wasGeneratedBy
used
used
used
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasDerivedFrom
wasAssociatedWith
wasInformedBy
wasAttributedTo
wasGeneratedBy
used
wasDerivedFrom
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasInformedBy
wasAttributedTo
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
used
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
wasGeneratedBy
wasGeneratedBy
actedOnBehalfOf
wasGeneratedBy
wasAttributedTo
wasAssociatedWith
actedOnBehalfOf
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
wasAssociatedWith
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
wasAttributedTo
used
used
used
wasAssociatedWith
used
used
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasAssociatedWith
wasInformedBy
wasInformedBy
wasAssociatedWith
wasAttributedTo
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasDerivedFrom
wasGeneratedBy
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasInformedBy
wasInformedBy
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
wasDerivedFrom
actedOnBehalfOf
actedOnBehalfOf
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasGeneratedBy
used
used
used
wasAttributedTo
wasAssociatedWith
wasGeneratedBy
wasGeneratedBy
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
actedOnBehalfOf
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

date, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
Legend
design
meaning
provenance: Agent
provenance: Entity
provenance: Activity
bold text
record those properties always
solid lining
record as detailed as possible
dashed lining
record without many details
grayed out
optional / not always existent
name
properties
name
properties
name
properties
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
HARVEST
hermes

version
HERMES cache
load

func, args, kwargs, source, time
harvest base plugin

settings
load

func, args, kwargs, source, time
harvest command

settings
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
process plugin

name, version, settings
merge strategies

strategies, time
process plugin

name, version, settings
.hermes/process/result/
codemeta.json

text, path uri, date created
merge strategies

strategies, time
merge strategies

start time, end time
write

start time, end time
.hermes/process/result/
expanded.json

text, path uri, date created
PROCESS
generate merge strategies

start time, end time
process base plugin

settings
generate merge strategies

start time, end time
process command

settings
.hermes/process/result/
context.json

text, path uri, date created
process plugin

name, version, settings
generate merge strategies

start time, end time
merge strategies

start time, end time
merged strategies

strategies, time
merged strategies

strategies, time
harvest plugin

name, version, settings
harvest source

path uri
.hermes/harvest/
{plugin_name}/codemeta.json

text, path uri, date created
harvested metadata

data
software-metadata

data, time
map

end time
write

start time, end time
.hermes/harvest/
{plugin_name}/expanded.json

text, path uri, date created
.hermes/harvest/
{plugin_name}/context.json

text, path uri, date created
load

func, args, kwargs, source, time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
software-metadata

data, time
load

start time, end time
used
used
wasAssociatedWith
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge value at key

key, strategy used
merge

start time, end time
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
software-metadata

time, data
software-metadata

time, data
reject/ replace/ ...
value with other value

start time, end time, strategy used
merge strategies

strategies, time
curate command

settings
software-metadata

data, time
load

start time, end time
curate base plugin

settings
curate plugin

name, version, settings
software-metadata

data, time
.hermes/curate/result/
codemeta.json

text, path uri, time created
write

start time, end time
.hermes/curate/result/
expanded.json

text, path uri, time created
.hermes/curate/result/
context.json

text, path uri, time created
CURATE
used
wasDerivedFrom
actedOnBehalfOf
used
used
used
wasDerivedFrom
wasDerivedFrom
wasGeneratedBy
actedOnBehalfOf
wasAttributedTo
wasAssociatedWith
wasAssociatedWith
actedOnBehalfOf
wasDerivedFrom
wasAttributedTo
wasAttributedTo
wasDerivedFrom
wasGeneratedBy
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
wasAssociatedWith
deposit command

settings
software-metadata

data, time
load

start time, end time
deposit base plugin

settings
deposit plugin

name, version, settings
mapped data for deposit

data, time
.hermes/deposit/
{deposit_plugin}/deposit.json

text, path uri, time created
write

start time, end time
DEPOSIT
map

start time, end time
updated metadata

data, time
.hermes/deposit/
{deposit_plugin}/result.json

text, path uri, time created
write

start time, end time
 \ No newline at end of file diff --git a/docs/adr/hermes-prov-diagram/hermes-prov.svg.license b/docs/adr/hermes-prov-diagram/hermes-prov.svg.license new file mode 100644 index 00000000..e4d2c6e9 --- /dev/null +++ b/docs/adr/hermes-prov-diagram/hermes-prov.svg.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) + +SPDX-License-Identifier: CC-BY-SA-4.0 \ No newline at end of file diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py index 51f2da08..b8fc536b 100644 --- a/src/hermes/commands/curate/base.py +++ b/src/hermes/commands/curate/base.py @@ -5,6 +5,8 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime +from typing import Optional from pydantic import BaseModel @@ -13,6 +15,7 @@ from hermes.model import SoftwareMetadata from hermes.model.context_manager import HermesContext from hermes.model.error import HermesValidationError +from hermes.model.provenance.ld_prov import ld_prov_list class HermesCuratePlugin(HermesPlugin): @@ -35,6 +38,17 @@ class HermesCurateCommand(HermesCommand): settings_class = CurateSettings def __call__(self, args: argparse.Namespace) -> None: + self.args = args + self.log.info("# Load provenance data from process step") + prov_doc = self.load_prov_doc() + if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("curate", self) + curate_command = prov_doc.get_hermes_command("curate") + curate_base_plugin = prov_doc.get_hermes_base_plugin("curate") + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + self.log.info("# Metadata curation") plugin_name = self.settings.plugin @@ -45,7 +59,9 @@ def __call__(self, args: argparse.Namespace) -> None: # load processed data ctx.prepare_step("process") try: + begin_load_at_time = datetime.datetime.now().isoformat() metadata = SoftwareMetadata.load_from_cache(ctx, "result") + end_load_at_time = datetime.datetime.now().isoformat() except Exception as e: self.log.critical( "## The data from the process step could not be loaded or is invalid for some reason.", @@ -54,6 +70,9 @@ def __call__(self, args: argparse.Namespace) -> None: raise HermesValidationError("The results of the process step are invalid.") from e ctx.finalize_step("process") + # save loaded metadata now, because it could be altered in curation + loaded_metadata_str = str(metadata.compact()) + self.log.info(f"## Load curation plugin {plugin_name}") # load plugin try: @@ -66,12 +85,111 @@ def __call__(self, args: argparse.Namespace) -> None: # run plugin try: curated_metadata = plugin_func(self, metadata) + end_curation_time = datetime.datetime.now().isoformat() except Exception as e: self.log.critical(f"## Unknown error while executing the {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError(f"Something went wrong while running the curate plugin {plugin_name}") from e self.log.info("## Store curated data") # store metadata + begin_store_at_time = datetime.datetime.now().isoformat() curated_metadata.write_to_cache(ctx, "result") + stored_at_time = datetime.datetime.now().isoformat() + + if prov_doc is not None: + curate_plugin = prov_doc.add_hermes_plugin("curate", plugin_name, plugin_func, self) + store_action_of_process = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [process_command.ref, hermes_cache.ref] and + "prov:wasInformedBy" in node + ))[0] + stored_results_of_process = [res.ref for res in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_of_process.ref] + ))] + load_action = prov_doc.add_activity(data={ + "schema:description": "loads the data from process step", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results_of_process, + "prov:startedAtTime": begin_load_at_time, + "prov:endedAtTime": end_load_at_time + }) + loaded_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "data loaded from process step", + "schema:text": loaded_metadata_str, # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": hermes_cache.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": stored_results_of_process, + "prov:generatedAtTime": end_load_at_time + }) + curated_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "curated metadata", + "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": [curate_plugin.ref, curate_base_plugin.ref, curate_command.ref], + "prov:wasInfluencedBy": curate_plugin.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": loaded_data.ref, + "prov:generatedAtTime": end_curation_time + }) + write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", + "prov:wasAssociatedWith": [curate_command.ref, hermes_cache.ref], + "prov:used": curated_data.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time + }) + # TODO: add more info + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(curated_metadata.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "codemeta.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": curated_metadata.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "context.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(curated_metadata.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "curate" / "result" / "expanded.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": curated_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value ctx.finalize_step("curate") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("process") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the process step could not be loaded. " + "Processing will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("process") diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index 57bed627..ace1e614 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -7,6 +7,8 @@ import abc import argparse +import datetime +from typing import Optional from pydantic import BaseModel @@ -15,6 +17,7 @@ from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata from hermes.model.error import HermesValidationError +from hermes.model.provenance.ld_prov import ld_prov_list class BaseDepositPlugin(HermesPlugin): @@ -23,36 +26,143 @@ class BaseDepositPlugin(HermesPlugin): TODO: describe workflow... needs refactoring to be less stateful! """ - def __call__(self, command: HermesCommand) -> None: + def __call__(self, command: HermesCommand, prov_doc: ld_prov_list) -> None: """Initiate the deposition process. This calls a list of additional methods on the class, none of which need to be implemented. """ self.command = command + target = command.settings.target self.ctx = HermesContext() self.ctx.prepare_step("deposit") self.ctx.prepare_step("curate") try: + start_of_load = datetime.datetime.now().isoformat() self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result") + end_of_load = datetime.datetime.now().isoformat() except Exception as e: raise HermesValidationError("The results of the curate step are invalid.") from e self.ctx.finalize_step("curate") + if prov_doc is not None: + plugin = prov_doc.add_hermes_plugin("deposit", target, self, command) + deposit_command = prov_doc.get_hermes_command("deposit") + curate_command = prov_doc.get_hermes_command("curate") + deposit_base_plugin = prov_doc.get_hermes_base_plugin("deposit") + hermes_cache = prov_doc.get_hermes_cache() + store_action_curate = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [curate_command.ref, hermes_cache.ref] and + "prov:used" in node and + len(node["prov:used"]) == 1 + ))[0] + results_curate = [item.ref for item in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action_curate.ref] + ))] + load_action = prov_doc.add_activity(data={ + "schema:description": "Loads the results of the curate step.", + "prov:used": results_curate, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_load, + "prov:endedAtTime": end_of_load + }) + loaded_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "data loaded from curate step", + "schema:text": str(self.metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": hermes_cache.ref, + "prov:wasGeneratedBy": load_action.ref, + "prov:wasDerivedFrom": results_curate, + "prov:generatedAtTime": end_of_load + }) + self.prepare() + start_of_map = datetime.datetime.now().isoformat() deposit = self.map_metadata() - with self.ctx[command.settings.target] as cache: + end_of_map = datetime.datetime.now().isoformat() + with self.ctx[target] as cache: + start_of_store = datetime.datetime.now().isoformat() cache["deposit"] = deposit + end_of_store = datetime.datetime.now().isoformat() + + if prov_doc is not None: + map_action = prov_doc.add_activity(data={ + "schema:description": "Maps the metadata to the format required by the deposition target.", + "prov:used": loaded_data.ref, + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": start_of_map, + "prov:endedAtTime": end_of_map + }) + mapped_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The metadata mapped to the format required by the deposition target.", + "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": map_action.ref, + "prov:wasDerivedFrom": loaded_data.ref, + "prov:generatedAtTime": end_of_load + }) + store_mapped_data = prov_doc.add_activity(data={ + "schema:description": "Stores the mapped metadata.", + "prov:used": mapped_data.ref, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_store, + "prov:endedAtTime": end_of_store + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The stored version of the mapped metadata.", + "schema:text": str(deposit), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (self.ctx.cache_dir / "deposit" / target / "deposit.json").absolute().as_uri(), + "prov:wasGeneratedBy": store_mapped_data.ref, + "prov:wasDerivedFrom": mapped_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": end_of_store + }) if self.is_initial_publication(): self.create_initial_version() else: self.create_new_version() - deposit = self.update_metadata() - with self.ctx[command.settings.target] as cache: - cache["result"] = deposit + updated_deposit = self.update_metadata() + end_of_update_map = datetime.datetime.now().isoformat() + with self.ctx[target] as cache: + start_of_second_store = datetime.datetime.now().isoformat() + cache["result"] = updated_deposit + end_of_second_store = datetime.datetime.now().isoformat() self.ctx.finalize_step("deposit") + + if prov_doc is not None: + updated_mapped_data = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The updated mapped metadata.", + "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "prov:wasInfluencedBy": plugin.ref, + "prov:wasDerivedFrom": mapped_data.ref, + "prov:generatedAtTime": end_of_update_map + }) + store_updated_mapped_data = prov_doc.add_activity(data={ + "schema:description": "Stores the mapped metadata.", + "prov:used": updated_mapped_data.ref, + "prov:wasAssociatedWith": [hermes_cache.ref, deposit_command.ref, deposit_base_plugin.ref], + "prov:startedAtTime": start_of_second_store, + "prov:endedAtTime": end_of_second_store + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The stored version of the updated mapped metadata.", + "schema:text": str(updated_deposit), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (self.ctx.cache_dir / "deposit" / target / "result.json").absolute().as_uri(), + "prov:wasGeneratedBy": store_updated_mapped_data.ref, + "prov:wasDerivedFrom": updated_mapped_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": end_of_second_store + }) + self.delete_artifacts() self.upload_artifacts() self.publish() @@ -138,6 +248,10 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("# Metadata deposition") self.args = args plugin_name = self.settings.target + prov_doc = self.load_prov_doc() + if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("deposit", self) self.log.info(f"## Load deposit plugin {plugin_name}") # load plugin @@ -150,9 +264,33 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"## Run deposit plugin {plugin_name}") # run plugin try: - plugin_func(self) + plugin_func(self, prov_doc) except HermesValidationError as e: self.log.critical(f"## Error while executing {plugin_name} plugin.", exc_info=1) raise HermesPluginRunError( f"Something went wrong while running the deposit plugin {self.settings.plugin}" ) from e + + if prov_doc is None: + return + + ctx = HermesContext() + ctx.prepare_step("deposit") + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("deposit") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("curate") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the curate step could not be loaded. " + "Deposition will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("curate") diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 0d3d9e5f..32550181 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -5,6 +5,9 @@ # SPDX-FileContributor: Michael Meinel import argparse +import datetime +from io import IOBase +from pathlib import Path from pydantic import BaseModel @@ -12,6 +15,7 @@ from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.context_manager import HermesContext from hermes.model import SoftwareMetadata +from hermes.model.provenance.ld_prov import ld_prov_list class HermesHarvestPlugin(HermesPlugin): @@ -19,10 +23,41 @@ class HermesHarvestPlugin(HermesPlugin): TODO: describe the harvesting process and how this is mapped to this plugin. """ + def __init__(self): + self.io_operations: list[tuple[dict, dict, dict]] = [] + super().__init__() def __call__(self, command: HermesCommand) -> SoftwareMetadata: pass + def load(self, func, source, *args, **kwargs): + source_metadata = {"schema:description": "metadata source"} + if isinstance(source, IOBase): + source_metadata["schema:url"] = Path(source.name).absolute().as_uri() + elif isinstance(source, Path): + source_metadata["schema:url"] = source.absolute().as_uri() + elif isinstance(source, str): + try: + source_metadata["schema:url"] = Path(source).absolute().as_uri() + except Exception: + source_metadata["schema:url"] = source + io_operation = { + "schema:description": "Load operation called with (" + f"{source_metadata['schema:url'] if 'schema:url' in source_metadata else str(source)}" + f"{', ' + str(args) if args else ''}{', ' + str(kwargs) if kwargs else ''}).", + "schema:name": f"{func.__module__}.{func.__qualname__}" + } + io_operation["prov:startedAtTime"] = datetime.datetime.now().isoformat() + result = func(source, *args, **kwargs) + io_operation["prov:endedAtTime"] = datetime.datetime.now().isoformat() + loaded_metadata = {"schema:description": "the loaded data", "schema:text": str(result)} + self.io_operations.append((source_metadata, io_operation, loaded_metadata)) + return result + + def write(): + # TODO: Is this needed? If yes, it needs to be implemented + pass + class HarvestSettings(BaseModel): """Generic harvesting settings.""" @@ -30,6 +65,32 @@ class HarvestSettings(BaseModel): sources: list[str] = [] +def remove_harvest_plugin_from_prov_doc(prov_doc: ld_prov_list, plugin: str) -> None: + plugin = prov_doc.get_hermes_plugin("harvest", plugin) + if plugin is None: + return + related = prov_doc.shallow_search(lambda node: ( + ("prov:wasAssociatedWith" in node and plugin.ref in node["prov:wasAssociatedWith"]) or + ("prov:wasAttributedTo" in node and plugin.ref in node["prov:wasAttributedTo"]) + )) + if len(related) == 0: + del prov_doc[plugin.index] + return + ids = [plugin.ref, *(rel.ref for rel in related)] + used_entities = [rel["prov:used"][0]["@id"] for rel in related if "prov:used" in rel] + related = prov_doc.shallow_search(lambda node: node["@id"] in used_entities) + related += prov_doc.shallow_search(lambda node: any( + (f"prov:{key}" in node and id in node[f"prov:{key}"]) for id in ids for key in [ + "wasAssociatedWith", "wasAttributedTo", "wasGeneratedBy", "used", "wasDerivedFrom", "wasInformedBy" + ] + )) + del prov_doc[plugin.index] + for item in related: + items = prov_doc.shallow_search(lambda node: ("@id" in node and node["@id"] == item["@id"])) + if len(items) == 1: + del prov_doc[items[0].index] + + class HermesHarvestCommand(HermesCommand): """ Harvest metadata from configured sources. """ @@ -37,9 +98,14 @@ class HermesHarvestCommand(HermesCommand): settings_class = HarvestSettings def __call__(self, args: argparse.Namespace) -> None: - self.log.info("# Metadata harvesting") self.args = args + self.log.info("# Load provenance from old harvest or create new document.") + prov_doc = self.init_provenance_document() + base_plugin = prov_doc.get_hermes_base_plugin("harvest") + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("harvest", self) + self.log.info("# Metadata harvesting") if len(self.settings.sources) == 0: self.log.critical("# No harvest plugin was configured to be run and loaded.") raise MisconfigurationError("No harvest plugin was configured to be run and loaded.") @@ -62,17 +128,119 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: - harvested_data = plugin_func(self) + harvested_data: SoftwareMetadata = plugin_func(self) except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue + returned_at_time = datetime.datetime.now().isoformat() self.log.info(f"### Store metadata harvested by {plugin_name} plugin") # store harvested data + begin_store_at_time = datetime.datetime.now().isoformat() harvested_data.write_to_cache(ctx, plugin_name) + stored_at_time = datetime.datetime.now().isoformat() harvested_any = True + remove_harvest_plugin_from_prov_doc(prov_doc, plugin_name) + + plugin = prov_doc.add_hermes_plugin("harvest", plugin_name, plugin_func, self) + plugin_io_operations = plugin_func.io_operations + outputs = [] + io_ops = [] + for plugin_io_operation in plugin_io_operations: + loaded_source = prov_doc.add_entity(data=plugin_io_operation[0]) + plugin_io_operation[1].update( + {"prov:wasAssociatedWith": [base_plugin.ref, plugin.ref], "prov:used": loaded_source.ref} + ) + io_op = prov_doc.add_activity(data=plugin_io_operation[1]) + plugin_io_operation[2].update({ + "prov:wasAttributedTo": plugin.ref, + "prov:wasDerivedFrom": loaded_source.ref, + "prov:wasGeneratedBy": io_op.ref + }) + loaded_data = prov_doc.add_entity(data=plugin_io_operation[2]) + outputs.append(loaded_data.ref) + io_ops.append(io_op.ref) + + map_activity = prov_doc.add_activity(data={ + "schema:description": "Maps the loaded data to the JSON-LD contexts vocabulary.", + "prov:wasInformedBy": io_ops, + "prov:used": outputs, + "prov:wasAssociatedWith": plugin.ref, + "prov:endedAtTime": returned_at_time + }) + data_output = prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "the harvested metadata", + "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": map_activity.ref, + "prov:wasDerivedFrom": outputs, + "prov:generatedAtTime": returned_at_time + }) + + write = prov_doc.add_activity(data={ + "schema:description": "Writes the harvested metadata into the HERMES cache.", + "prov:wasAssociatedWith": [ + prov_doc.get_hermes_command("harvest").ref, + prov_doc.get_hermes_cache().ref, + plugin.ref + ], + "prov:used": data_output.ref, + "prov:wasInformedBy": map_activity.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the harvested metadata.", + "schema:text": str(harvested_data.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "codemeta.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the harvested metadata.", + "schema:text": str({"@context": harvested_data.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "context.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the harvested metadata.", + "schema:text": str(harvested_data.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "harvest" / plugin_name / "expanded.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": data_output.ref, + "prov:wasAttributedTo": prov_doc.get_hermes_cache().ref, + "prov:generatedAtTime": stored_at_time + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step('harvest') if not harvested_any: self.log.critical("No harvest plugin ran successfully.") raise HermesPluginRunError("No harvest plugin ran successfully.") + + def init_provenance_document(self) -> ld_prov_list: + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except KeyError: + pass + prov_doc = ld_prov_list() + prov_doc.init_hermes_agents() + return prov_doc diff --git a/src/hermes/commands/harvest/cff.py b/src/hermes/commands/harvest/cff.py index 5a2d16c1..f2b648e6 100644 --- a/src/hermes/commands/harvest/cff.py +++ b/src/hermes/commands/harvest/cff.py @@ -43,7 +43,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic 'Aborting harvesting for this metadata source.') # Read the content - cff_data = cff_file.read_text() + cff_data = self.load(pathlib.Path.read_text, cff_file) cff_dict = self._load_cff_from_file(cff_data) if command.settings.cff.enable_validation: diff --git a/src/hermes/commands/harvest/codemeta.py b/src/hermes/commands/harvest/codemeta.py index 3dc84296..07645647 100644 --- a/src/hermes/commands/harvest/codemeta.py +++ b/src/hermes/commands/harvest/codemeta.py @@ -34,7 +34,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic ) # Read the content - codemeta_str = codemeta_file.read_text() + codemeta_str = self.load(pathlib.Path.read_text, codemeta_file) if not self._validate(codemeta_file): raise HermesValidationError(codemeta_file) diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 725f6487..eda4962c 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -5,22 +5,25 @@ # SPDX-FileContributor: Michael Meinel import argparse -from typing import Union +import datetime +from typing import Optional from pydantic import BaseModel from hermes.commands.base import HermesCommand, HermesPlugin +from hermes.commands.harvest.base import remove_harvest_plugin_from_prov_doc from hermes.error import HermesPluginRunError, MisconfigurationError from hermes.model.api import SoftwareMetadata from hermes.model.context_manager import HermesContext from hermes.model.merge.action import MergeAction from hermes.model.merge.container import ld_merge_dict +from hermes.model.provenance.ld_prov import ld_prov_list class HermesProcessPlugin(HermesPlugin): """ Base plugin that defines additional merge strategies.""" - def __call__(self, command: HermesCommand) -> dict[Union[str, None], dict[Union[str, None], MergeAction]]: + def __call__(self, command: HermesCommand) -> dict[Optional[str], dict[Optional[str], MergeAction]]: pass @@ -38,8 +41,17 @@ class HermesProcessCommand(HermesCommand): settings_class = ProcessSettings def __call__(self, args: argparse.Namespace) -> None: + self.args = args + self.log.info("# Load provenance data from harvest step") + prov_doc = self.load_prov_doc() + if prov_doc is not None: + prov_doc.add_hermes_settings(self) + prov_doc.add_settings_to_command("process", self) + process_command = prov_doc.get_hermes_command("process") + hermes_cache = prov_doc.get_hermes_cache() + self.log.info("# Metadata processing") - merged_doc = ld_merge_dict([{}]) + merged_doc = ld_merge_dict([{}], prov_doc) if not self.settings.plugins: self.log.critical( @@ -56,6 +68,7 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Load and run the plugins") any_strategies_loaded = False + strategy_action, merged_strategies = None, None # add the strategies from the plugins for plugin_name in reversed(self.settings.plugins): self.log.info(f"### Load {plugin_name} plugin") @@ -69,16 +82,55 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Run {plugin_name} plugin") # run plugin try: + generate_strategies_start = datetime.datetime.now().isoformat() additional_strategies = plugin_func(self) + generate_strategies_end = datetime.datetime.now().isoformat() except Exception: self.log.exception(f"### Unknown error while executing the {plugin_name} plugin, skipping it now.") continue self.log.info(f"### Add the strategies to the merge document {plugin_name} plugin") # add strategies to the merge document + merge_strategies_start = datetime.datetime.now().isoformat() merged_doc.add_strategy(additional_strategies) + merge_strategies_end = datetime.datetime.now().isoformat() any_strategies_loaded = True + if prov_doc is None: + continue + plugin = prov_doc.add_hermes_plugin("process", plugin_name, plugin_func, self) + new_strategy_generation = prov_doc.add_activity(data={ + "schema:description": "generate new merge strategies", + "prov:wasAssociatedWith": plugin.ref, + "prov:startedAtTime": generate_strategies_start, + "prov:endedAtTime": generate_strategies_end + }) + new_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "schema:description": f"new merge strategies of plugin {plugin_name}", + "prov:wasAttributedTo": plugin.ref, + "prov:wasGeneratedBy": new_strategy_generation.ref, + "prov:generatedAtTime": generate_strategies_end + }) + if merged_strategies is None: + merged_strategies = new_strategies + strategy_action = new_strategy_generation + continue + strategy_action = prov_doc.add_activity(data={ + "schema:description": "merging the new strategies into the others", + "prov:used": [merged_strategies.ref, new_strategies.ref], + "prov:wasInformedBy": [strategy_action.ref, new_strategy_generation.ref], + "prov:wasAssociatedWith": process_command.ref, + "prov:startedAtTime": merge_strategies_start, + "prov:endedAtTime": merge_strategies_end + }) + merged_strategies = prov_doc.add_entity(data={ # TODO: record strategies + "schema:description": "the merge strategies of multiple plugins merged together", + "prov:wasDerivedFrom": [merged_strategies.ref, new_strategies.ref], + "prov:wasGeneratedBy": strategy_action.ref, + "prov:wasAttributedTo": process_command.ref, + "prov:generatedAtTime": merge_strategies_end + }) + if not any_strategies_loaded: self.log.critical("## No process plugin was ran successfully.") raise HermesPluginRunError("No process plugin was ran successfully.") @@ -93,21 +145,73 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info(f"### Load data from {harvester} plugin") # load data from harvester try: + load_start = datetime.datetime.now().isoformat() metadata = SoftwareMetadata.load_from_cache(ctx, harvester) + load_end = datetime.datetime.now().isoformat() except Exception: # skip this harvester when the data is invalid + if prov_doc is not None: + remove_harvest_plugin_from_prov_doc(prov_doc, harvester) self.log.exception( f"### The data from the harvester {harvester} could not be loaded or is invalid, skipping it now." ) continue + if prov_doc is not None: + harvest_plugin = prov_doc.get_hermes_plugin("harvest", harvester) + harvest_command = prov_doc.get_hermes_command("harvest") + store_action = prov_doc.shallow_search(lambda node: ( + "prov:wasAssociatedWith" in node and + node["prov:wasAssociatedWith"] == [harvest_plugin.ref, hermes_cache.ref, harvest_command.ref] + ))[0] + stored_results = [ + result.ref for result in prov_doc.shallow_search(lambda node: ( + "prov:wasGeneratedBy" in node and node["prov:wasGeneratedBy"] == [store_action.ref] + )) + ] + new_action = prov_doc.add_activity(data={ # load of new data + "schema:description": f"loads the data from {harvester} plugin", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": stored_results, + "prov:startedAtTime": load_start, + "prov:endedAtTime": load_end + }) + new_data = prov_doc.add_entity(data={ # new data to be merged + "@type": "schema:CreativeWork", + "schema:description": f"data loaded from {harvester} plugin", + "schema:text": str(metadata.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": [process_command.ref, hermes_cache.ref], + "prov:wasGeneratedBy": new_action.ref, + "prov:wasDerivedFrom": stored_results, + "prov:generatedAtTime": load_end + }) + if merged_any: + # One pass must have been completed already. + new_action = prov_doc.add_activity(data={ + "schema:description": "merges the old data object with the new data", + "prov:used": [last_data.ref, new_data.ref], + "prov:wasInformedBy": [last_action.ref, new_action.ref], + "prov:wasAssociatedWith": process_command.ref + }) # initial merge action of the merge + merged_doc.prov_objects = [new_action, new_data, last_data] # set the starting objects of the merge + self.log.info(f"### Merge data from {harvester} plugin") # merge data into the merge dict try: + merge_start = datetime.datetime.now().isoformat() merged_doc.update(metadata) + merge_end = datetime.datetime.now().isoformat() except Exception as e: + # TODO: Maybe this state is recoverable by starting over again and skipping this plugin. self.log.critical(f"### Merging the data from {harvester} plugin resulted in an error.", exc_info=True) raise RuntimeError(f"Merging the data from {harvester} plugin failed.") from e + + if prov_doc is not None: + if merged_any: + new_action["prov:startedAtTime"] = merge_start + new_action["prov:endedAtTime"] = merge_end + last_action = merged_doc.prov_objects[0] if merged_any else new_action + last_data = merged_doc.prov_objects[2] if merged_any else new_data merged_any = True # error if nothing was merged @@ -118,10 +222,75 @@ def __call__(self, args: argparse.Namespace) -> None: self.log.info("## Store processed metadata") # store processed data ctx.prepare_step("process") + begin_store_at_time = datetime.datetime.now().isoformat() with ctx["result"] as result_ctx: result_ctx["codemeta"] = merged_doc.compact() result_ctx["context"] = {"@context": merged_doc.full_context} result_ctx["expanded"] = merged_doc.ld_value + stored_at_time = datetime.datetime.now().isoformat() + + if prov_doc is not None: + write = prov_doc.add_activity(data={ + "schema:description": "Writes the processed metadata into the HERMES cache.", + "prov:wasAssociatedWith": [process_command.ref, hermes_cache.ref], + "prov:used": last_data.ref, + "prov:wasInformedBy": last_action.ref, + "prov:startedAtTime": begin_store_at_time, + "prov:endedAtTime": stored_at_time + }) + # TODO: add more info + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The compacted version of the processed metadata.", + "schema:text": str(merged_doc.compact()), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "codemeta.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": last_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The context of the processed metadata.", + "schema:text": str({"@context": merged_doc.full_context}), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "context.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": last_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": "The expanded version of the processed metadata.", + "schema:text": str(merged_doc.ld_value), # TODO: maybe "prov:value" instead? + "schema:encodingFormat": "application/json", + "schema:url": (ctx.cache_dir / "process" / "result" / "expanded.json").absolute().as_uri(), + "prov:wasGeneratedBy": write.ref, + "prov:wasDerivedFrom": last_data.ref, + "prov:wasAttributedTo": hermes_cache.ref, + "prov:generatedAtTime": stored_at_time + }) + + with ctx["provenance"] as cache: + cache["result"] = prov_doc.ld_value + ctx.finalize_step("process") ctx.finalize_step("harvest") + + def load_prov_doc(self) -> Optional[ld_prov_list]: + ctx = HermesContext() + ctx.prepare_step("harvest") + with ctx["provenance"] as cache: + try: + return ld_prov_list.load_ld_prov_list(cache["result"]) + except Exception: + self.log.warning( + "The provenance data from the harvest step could not be loaded. " + "Processing will proceed without collecting provenance data.", + exc_info=1 + ) + finally: + ctx.finalize_step("harvest") diff --git a/src/hermes/model/merge/container.py b/src/hermes/model/merge/container.py index e9cc03a2..c2330029 100644 --- a/src/hermes/model/merge/container.py +++ b/src/hermes/model/merge/container.py @@ -7,9 +7,11 @@ from __future__ import annotations +import datetime from typing import TYPE_CHECKING, Any, Callable, Optional, Union from typing_extensions import Self +from hermes.model.provenance.ld_prov import ld_prov_list from hermes.model.types import ld_container, ld_context, ld_dict, ld_list from hermes.model.types.ld_container import ( BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT, JSON_LD_VALUE, TIME_TYPE @@ -50,6 +52,8 @@ def _to_python( if isinstance(value, ld_dict) and not isinstance(value, ld_merge_dict): value = ld_merge_dict( value.ld_value, + self.prov_doc, + self.prov_objects, parent=value.parent, key=value.key, index=value.index, @@ -60,6 +64,8 @@ def _to_python( if isinstance(value, ld_list) and not isinstance(value, ld_merge_list): value = ld_merge_list( value.ld_value, + self.prov_doc, + self.prov_objects, parent=value.parent, key=value.key, index=value.index, @@ -82,6 +88,8 @@ class ld_merge_list(_ld_merge_container, ld_list): def __init__( self: "ld_merge_list", data: Union[list[str], list[dict[str, EXPANDED_JSON_LD_VALUE]]], + prov_doc: ld_prov_list = None, + prov_objects: list[ld_dict] = 3*[None], *, parent: Optional[ld_container] = None, key: Optional[str] = None, @@ -108,6 +116,8 @@ def __init__( super().__init__(data, parent=parent, key=key, index=index, context=context) self.strategies = strategies + self.prov_doc = prov_doc + self.prov_objects = prov_objects class ld_merge_dict(_ld_merge_container, ld_dict): @@ -123,6 +133,8 @@ class ld_merge_dict(_ld_merge_container, ld_dict): def __init__( self: Self, data: list[dict[str, EXPANDED_JSON_LD_VALUE]], + prov_doc: ld_prov_list = None, + prov_objects: list[ld_dict] = 3*[None], *, parent: Optional[Union[ld_dict, ld_list]] = None, key: Optional[str] = None, @@ -154,6 +166,8 @@ def __init__( # add strategies self.strategies = strategies + self.prov_doc = prov_doc + self.prov_objects = prov_objects def update_context( self: Self, other_context: Union[list[Union[str, JSON_LD_CONTEXT_DICT]], None] @@ -224,10 +238,46 @@ def __setitem__(self: Self, key: str, value: Union[JSON_LD_VALUE, BASIC_TYPE, TI ``self[key]``. """ # create the new item if self[key] and value have to be merged. + merge_start = datetime.datetime.now().isoformat() if key in self: - value = self._merge_item(key, value) + if self.prov_objects[0] is not None: + last_merged_data = self.prov_objects[2] + merge_activity, value = self._merge_item(key, value) + if self.prov_objects[0] is not None: + create_new_merged_data = last_merged_data is self.prov_objects[2] + elif self.prov_objects[0] is not None: + merge_activity = self.prov_doc.add_activity(data={ + "schema:name": f"merge values at {str(self.path+[key])}", + "schema:description": f"inserting value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point", + "prov:used": {"@list": [self.prov_objects[2].ref, self.prov_objects[1].ref]}, + "prov:wasInformedBy": self.prov_objects[0].ref + }) + create_new_merged_data = True # update the entry of self[key] super().__setitem__(key, value) + merge_end = datetime.datetime.now().isoformat() + if self.prov_objects[0] is None: + return + if merge_activity is not None: + merge_activity["prov:startedAtTime"] = merge_start + merge_activity["prov:endedAtTime"] = merge_end + self.prov_objects[0] = merge_activity + if create_new_merged_data: + outer_most_parent = self + while outer_most_parent.parent != None: + outer_most_parent = outer_most_parent.parent + self.prov_objects[2] = self.prov_doc.add_entity(data={ + "@type": "schema:CreativeWork", + "schema:description": f"software metadata after merge of values at {str(self.path+[key])}", + "schema:text": str(outer_most_parent.compact()), # TODO: maybe "prov:value" instead? + "prov:wasAttributedTo": self.prov_doc.get_hermes_command("process").ref, + "prov:wasGeneratedBy": merge_activity.ref, + "prov:wasDerivedFrom": {"@list": [self.prov_objects[1].ref, self.prov_objects[2].ref]}, + "prov:generatedAtTime": merge_end + }) + else: + self.prov_objects[2]["prov:wasGeneratedBy"].append(merge_activity.ref) def match( self: Self, @@ -281,7 +331,23 @@ def _merge_item( merger = strategy.get(key, strategy.get(None, None)) if merger is None: raise MergeError(f"Can't merge, no strategy found for key '{key}'.") - return merger.merge(self, [*self.path, key], self[key], value) + if self.prov_objects[0] is not None: + merge_activity = self.prov_doc.add_activity(data={ + "schema:name": f"merge values at {str(self.path+[key])}", + "schema:description": f"merge value in the second 'used' value at {str(self.path+[key])} into the " + "first 'used' value at the same point using the third 'used' value", + "prov:wasAssociatedWith": self.prov_doc.get_hermes_command("process").ref, + "prov:used": {"@list": [ + self.prov_objects[2].ref, + self.prov_objects[1].ref, + f"{merger.merge.__module__}.{merger.merge.__qualname__}" + ]}, + "prov:wasInformedBy": self.prov_objects[0].ref + }) + self.prov_objects[0] = merge_activity + else: + merge_activity = None + return merge_activity, merger.merge(self, [*self.path, key], self[key], value) def _add_related( self: Self, rel: str, key: str, value: Union[BASIC_TYPE, TIME_TYPE, ld_dict, ld_list] diff --git a/src/hermes/model/provenance/ld_prov.py b/src/hermes/model/provenance/ld_prov.py new file mode 100644 index 00000000..ba86c274 --- /dev/null +++ b/src/hermes/model/provenance/ld_prov.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: 2026 German Aerospace Center (DLR) +# +# SPDX-License-Identifier: Apache-2.0 + +# SPDX-FileContributor: Michael Fritzsche + +from importlib.metadata import metadata +from typing import Optional, Union +from typing_extensions import Self + +from hermes import utils +from hermes.commands.base import HermesCommand, HermesPlugin +from hermes.model.types import ld_dict, ld_list +from hermes.model.types.ld_container import BASIC_TYPE, EXPANDED_JSON_LD_VALUE, JSON_LD_CONTEXT_DICT +from hermes.model.types.ld_context import ALL_CONTEXTS, iri_map + + +class ld_prov_list(ld_list): + NODE_IRI_FORMAT = "_:{type}/{index}" + HERMES_ID = f"https://doi.org/{utils.hermes_doi}" + HERMES_CACHE_ID = "_:hermes/cache" + HERMES_COMMAND_ID_FORMAT = "_:hermes/command/{step}" + HERMES_PLUGIN_ID_FORMAT = "_:hermes/plugin/{step}/{name}" + HERMES_BASE_PLUGIN_ID_FORMAT = "_:hermes/base_plugin/{step}" + PROV_DOC_IRI = iri_map['hermes-rt', "graph"] + INDICES = {} + + def __init__( + self: Self, + data: EXPANDED_JSON_LD_VALUE = [{"@graph": []}], + *, + parent: Optional[Union[ld_dict, ld_list]] = None, + key: Optional[str] = PROV_DOC_IRI, + index: Optional[int] = None, + context: Optional[list[Union[str, JSON_LD_CONTEXT_DICT]]] = ALL_CONTEXTS + ) -> None: + super().__init__(data, parent=parent, key=key, index=index, context=context) + + @classmethod + def load_ld_prov_list(cls, data) -> "ld_prov_list": + if cls.INDICES != {}: + raise RuntimeError("Only zero or one objects of class 'ld_prov_list' may exist at every point in time.") + prov_list = cls.from_list( + data[0]["@graph"], key=cls.PROV_DOC_IRI, context=ALL_CONTEXTS, container_type="@graph" + ) + for item in prov_list: + if not ("@id" in item and item["@id"].startswith("_:")): + continue + item_id = item["@id"][2:].split("/") + if not (len(item_id) == 2 and item_id[1].isnumeric()): + continue + if cls.INDICES.get(item_id[0], 0) < int(item_id[1]): + cls.INDICES[item_id[0]] = int(item_id[1]) + return prov_list + + def next_node_iri(self, type) -> str: + if type not in ld_prov_list.INDICES: + ld_prov_list.INDICES[type] = 0 + ld_prov_list.INDICES[type] += 1 + return self.NODE_IRI_FORMAT.format(type=type, index=ld_prov_list.INDICES[type]) + + def add_activity(self, *, data={}) -> ld_dict: + self.append(data) + activity = self[-1] + if "@type" not in data: + activity["@type"] = ["prov:Activity", "schema:Action"] + else: + activity["@type"].extend(["prov:Activity", "schema:Action"]) + if "@id" not in data: + activity["@id"] = self.next_node_iri("Activity") + return activity + + def add_agent(self, *, data={}) -> ld_dict: + self.append(data) + agent = self[-1] + if "@type" not in data: + agent["@type"] = ["prov:Agent", "schema:SoftwareApplication"] + else: + agent["@type"].extend(["prov:Agent", "schema:SoftwareApplication"]) + if "@id" not in data: + agent["@id"] = self.next_node_iri("Agent") + return agent + + def add_entity(self, *, data={}) -> ld_dict: + self.append(data) + entity = self[-1] + if "@type" not in data: + entity["@type"] = ["prov:Entity", "schema:Thing"] + else: + entity["@type"].extend(["prov:Entity", "schema:Thing"]) + if "@id" not in data: + entity["@id"] = self.next_node_iri("Entity") + return entity + + def init_hermes_agents(self) -> None: + hermes = self.add_agent(data={ + "@id": ld_prov_list.HERMES_ID, + "@type": "schema:SoftwareApplication", + "schema:name": utils.hermes_name, + "schema:version": utils.hermes_version, + "schema:url": [*set(utils.hermes_urls.values())] + }) + self.add_agent(data={ + "@id": ld_prov_list.HERMES_CACHE_ID, + "@type": "schema:SoftwareApplication", + "schema:name": utils.hermes_name + " cache", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": hermes.ref + }) + for step in ["harvest", "process", "curate", "deposit", "postprocess"]: + command = self.add_agent(data={ + "@id": ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step), + "@type": "schema:SoftwareApplication", + "schema:name": f"{utils.hermes_name} {step} command", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": hermes.ref + }) + self.add_agent(data={ + "@id": ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step), + "@type": "schema:SoftwareApplication", + "schema:name": f"{utils.hermes_name} {step} base plugin", + "schema:version": utils.hermes_version, + "prov:actedOnBehalfOf": command.ref + }) + + def add_hermes_settings(self, command: HermesCommand) -> None: + hermes = self.get_hermes() + hermes.emplace("schema:supportingData") + hermes["schema:supportingData"].append({ + "@type": "schema:DataFeed", + "schema:dataFeedElement": [ + { + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + ], + "schema:description": "setting provided by command line (or its default value)" + } + for name, value in [ + ("path", command.args.path.absolute().as_uri()), + ("config", command.args.config.absolute().as_uri()), + ("options", str(command.args.options)) + ] + ], + "schema:description": f"options for run {len(hermes['schema:supportingData']) + 1} of some hermes step" + }) + for name, values in command.root_settings.model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + hermes["schema:supportingData"][-1]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ], + "schema:description": "setting loaded from the config file" + }) + + def add_settings_to_command(self, step: str, command: HermesCommand) -> None: + command_prov = self.get_hermes_command(step) + command_prov.emplace("schema:supportingData") + command_prov["schema:supportingData"].append({ + "@type": "schema:DataFeed", + "schema:dataFeedElement": [], + "schema:description": f"options for run {len(command_prov['schema:supportingData']) + 1} of step {step} out" + f" of {len(self.get_hermes()['schema:supportingData'])} runs of some hermes step" + }) # Needs add_hermes_settings to be called before add_settings_to_command is called! + for name, values in command.settings.model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + command_prov["schema:supportingData"][-1]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ] + }) + + def add_hermes_plugin(self, step: str, name: str, plugin: HermesPlugin, command: HermesCommand) -> ld_dict: + data = { + "@id": ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name), + "@type": "schema:SoftwareApplication", + "schema:name": f"{plugin.__module__}.{plugin.__class__.__qualname__}", + "schema:description": f"{utils.hermes_name} {step} plugin '{name}'", + "schema:supportingData": { + "@type": "schema:DataFeed", + "schema:dataFeedElement": [] + }, + "prov:actedOnBehalfOf": self.get_hermes_base_plugin(step).ref + } + try: + for name, values in getattr(command.settings, name).model_dump(mode="json").items(): + if not isinstance(values, list): + values = [values] + data["schema:supportingData"]["schema:dataFeedElement"].append({ + "@type": "schema:DataFeedItem", + "schema:name": name, + "schema:item": [ + { + "@type": "schema:Item", + "schema:description": value + } + for value in values + ] + }) + except Exception: + del data["schema:supportingData"] + try: + data["schema:softwareVersion"] = metadata(plugin.__module__)["version"] + except Exception: + pass + node = self.add_agent(data=data) + return node + + def shallow_search(self, query) -> list[ld_dict]: + return [item for item in self if query(item)] + + def get_hermes(self) -> ld_dict: + return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_ID))[0] + + def get_hermes_cache(self) -> ld_dict: + return self.shallow_search(lambda node: ("@id" in node and node["@id"] == ld_prov_list.HERMES_CACHE_ID))[0] + + def get_hermes_base_plugin(self, step) -> ld_dict: + return self.shallow_search(lambda node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_BASE_PLUGIN_ID_FORMAT.format(step=step) + ))[0] + + def get_hermes_plugin(self, step, name) -> Union[ld_dict, None]: + search_result = self.shallow_search(lambda node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_PLUGIN_ID_FORMAT.format(step=step, name=name) + )) + if search_result: + return search_result[0] + return None + + def get_hermes_command(self, step) -> ld_dict: + return self.shallow_search(lambda node: ( + "@id" in node and node["@id"] == ld_prov_list.HERMES_COMMAND_ID_FORMAT.format(step=step) + ))[0] diff --git a/src/hermes/model/types/ld_container.py b/src/hermes/model/types/ld_container.py index abc37fa9..10d2a82c 100644 --- a/src/hermes/model/types/ld_container.py +++ b/src/hermes/model/types/ld_container.py @@ -316,7 +316,7 @@ def compact( COMPACTED_JSON_LD_VALUE: The compacted version of selfs JSON-LD representation. """ return self.ld_proc.compact( - self.ld_value, context or self.context, {"documentLoader": bundled_loader, "skipExpand": True} + self.ld_value, context or self.full_context, {"documentLoader": bundled_loader, "skipExpand": True} ) def to_python(self): diff --git a/src/hermes/model/types/ld_list.py b/src/hermes/model/types/ld_list.py index 14331472..01a1c265 100644 --- a/src/hermes/model/types/ld_list.py +++ b/src/hermes/model/types/ld_list.py @@ -15,6 +15,7 @@ from typing_extensions import Self from .ld_container import ( + COMPACTED_JSON_LD_VALUE, ld_container, JSON_LD_CONTEXT_DICT, EXPANDED_JSON_LD_VALUE, @@ -23,6 +24,7 @@ TIME_TYPE, BASIC_TYPE, ) +from .pyld_util import bundled_loader if TYPE_CHECKING: from .ld_dict import ld_dict @@ -548,6 +550,45 @@ def to_python(self: Self) -> list[Union[BASIC_TYPE, TIME_TYPE, PYTHONIZED_LD_CON for item in self ] + def compact( + self: Self, context: Optional[Union[list[Union[JSON_LD_CONTEXT_DICT, str]], JSON_LD_CONTEXT_DICT, str]] = None + ) -> COMPACTED_JSON_LD_VALUE: + """ + Returns the compacted version of the given ld_list using its context only if none was supplied. + The returned object is of the form `{"@context": the_context, container_type: compacted_content}`. + + Args: + context (list[JSON_LD_CONTEXT_DICT | str] | JSON_LD_CONTEXT_DICT | str | None): + The context to use for the compaction. If None the context of self is used. + + Returns: + COMPACTED_JSON_LD_VALUE: The compacted version of selfs JSON-LD representation. + """ + # compact the ld_list standalone if necessary + if self.key is None: + return self.ld_proc.compact( + self.ld_value, context or self.full_context, {"documentLoader": bundled_loader, "skipExpand": True} + ) + # compact the ld_list within a temporary dictionary + temp_dict = self.ld_proc.compact( + [{self.ld_proc.expand_iri(self.active_ctx, self.key): self.ld_value}], + context or self.full_context, + {"documentLoader": bundled_loader, "skipExpand": True} + ) + context = temp_dict["@context"] + temp_container = temp_dict[ + self.ld_proc.compact_iri(self.active_ctx, self.ld_proc.expand_iri(self.active_ctx, self.key)) + ] + if self.container_type != "@set": + return { + "@context": context, + **temp_container + } + return { + "@context": context, + "@set": temp_container if isinstance(temp_container, list) else [temp_container] + } + @classmethod def is_ld_list(cls: type[Self], ld_value: Any) -> bool: """ @@ -589,7 +630,7 @@ def from_list( key: Optional[str] = None, context: Optional[Union[str, JSON_LD_CONTEXT_DICT, list[Union[str, JSON_LD_CONTEXT_DICT]]]] = None, container_type: str = "@set" - ) -> ld_list: + ) -> Self: """ Creates a ld_list from the given list with the given parent, key, context and container_type.\n Note that only container_type '@set' is valid for key '@type'.\n