From 255fce164bc202b46d7846c4fd45e21f2c3f91b9 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:41:31 +0900 Subject: [PATCH 1/3] Preserve browser evidence when a notebook step fails --- scripts/playwright.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/playwright.py b/scripts/playwright.py index 9334f26..459404b 100644 --- a/scripts/playwright.py +++ b/scripts/playwright.py @@ -72,6 +72,12 @@ async def run_pw(f, last_path=default_last_path, screenshot=True, permissions=No "type": msg.type, "text": msg.text })) + page.on("pageerror", lambda error: console_messages.append({ + "timestamp": time.time(), + "url": page.url, + "type": "pageerror", + "text": str(error) + })) current_pages.append(page) current_time = time.time() @@ -214,7 +220,6 @@ async def _finish_pw_context(screenshot=False, last_path=None): if current_contexts is None or len(current_contexts) == 0: return current_context, current_pages = current_contexts[-1] - current_contexts = current_contexts[:-1] os.makedirs(last_path or default_last_path, exist_ok=True) timeout_on_screenshot = False if screenshot and current_pages is not None and len(current_pages) > 0: @@ -226,6 +231,7 @@ async def _finish_pw_context(screenshot=False, last_path=None): timeout_on_screenshot = True if timeout_on_screenshot: return + current_contexts = current_contexts[:-1] pages = list(current_pages) for page in pages: if not page.is_closed(): From 6cfcf8e78acad43dff3de31d38182d9334724a17 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:07:35 +0900 Subject: [PATCH 2/3] Wait for the IdP dropdown image before clicking --- ...6-\343\203\255\343\202\260\343\202\244\343\203\263.ipynb" | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git "a/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\343\203\255\343\202\260\343\202\244\343\203\263.ipynb" "b/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\343\203\255\343\202\260\343\202\244\343\203\263.ipynb" index 46e97e3..8ad9405 100644 --- "a/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\343\203\255\343\202\260\343\202\244\343\203\263.ipynb" +++ "b/\343\203\206\343\202\271\343\203\210\346\211\213\351\240\206-\343\203\255\343\202\260\343\202\244\343\203\263.ipynb" @@ -819,7 +819,10 @@ "async def _step(page):\n", " if idp_name == 'FakeCAS':\n", " return\n", - " await page.locator('//*[@id = \"dropdown_img\"]').click()\n", + " dropdown = page.locator('//*[@id = \"dropdown_img\"]')\n", + " await expect(dropdown).to_have_js_property('complete', True, timeout=transition_timeout)\n", + " await expect(dropdown).not_to_have_js_property('naturalWidth', 0, timeout=transition_timeout)\n", + " await dropdown.click()\n", "\n", " # IdPが要素として作成されることを確認\n", " locator = page.locator(f'//*[@class = \"list_idp\" and text() = \"{idp_name}\"]')\n", From 9dc008d3b067f0f858c42dc11a18f326563e873d Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:26:30 +0900 Subject: [PATCH 3/3] Add Shibboleth login coverage to the existing E2E matrix --- .github/pull_request_template.md | 6 + .github/scripts/generate_ci_config.sh | 28 ++- .github/scripts/setup_rdm.sh | 18 +- .github/scripts/setup_shibboleth.py | 145 ++++++++++++++++ .github/scripts/setup_test_data.py | 21 ++- .github/workflows/e2e-test.yml | 161 +++++++++++++++++- docker/shibboleth/cas/institutions-auth.xsl | 20 +++ docker/shibboleth/idp/Dockerfile | 22 +++ .../shibboleth/idp/conf/attribute-filter.xml | 15 ++ .../idp/conf/attribute-resolver.xml | 29 ++++ .../idp/conf/authn/password-authn-config.xml | 9 + .../idp/conf/metadata-providers.xml | 4 + docker/shibboleth/idp/conf/relying-party.xml | 19 +++ docker/shibboleth/idp/consent.css | 16 ++ docker/shibboleth/idp/install.properties | 1 + docker/shibboleth/idp/start.sh | 27 +++ docker/shibboleth/sp/attribute-map.xml | 11 ++ docker/shibboleth/sp/discovery/index.html | 18 ++ docker/shibboleth/sp/headers.conf | 21 +++ docker/shibboleth/sp/httpd.conf | 40 +++++ docker/shibboleth/sp/shibboleth2.xml | 16 ++ 21 files changed, 640 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/setup_shibboleth.py create mode 100644 docker/shibboleth/cas/institutions-auth.xsl create mode 100644 docker/shibboleth/idp/Dockerfile create mode 100644 docker/shibboleth/idp/conf/attribute-filter.xml create mode 100644 docker/shibboleth/idp/conf/attribute-resolver.xml create mode 100644 docker/shibboleth/idp/conf/authn/password-authn-config.xml create mode 100644 docker/shibboleth/idp/conf/metadata-providers.xml create mode 100644 docker/shibboleth/idp/conf/relying-party.xml create mode 100644 docker/shibboleth/idp/consent.css create mode 100644 docker/shibboleth/idp/install.properties create mode 100644 docker/shibboleth/idp/start.sh create mode 100644 docker/shibboleth/sp/attribute-map.xml create mode 100644 docker/shibboleth/sp/discovery/index.html create mode 100644 docker/shibboleth/sp/headers.conf create mode 100644 docker/shibboleth/sp/httpd.conf create mode 100644 docker/shibboleth/sp/shibboleth2.xml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fae850b..7a21b80 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,6 +13,12 @@ - OSF_IMAGE: - EMBER_IMAGE: - CAS_IMAGE: + +- CAS_REPOSITORY: +- CAS_BRANCH: +- SP_REPOSITORY: +- SP_BRANCH: + - MFR_IMAGE: - WB_IMAGE: - EXCLUDE_NOTEBOOKS: diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index 86d960c..16d645f 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -3,7 +3,7 @@ set -xeuo pipefail if [[ $# -lt 2 ]]; then cat >&2 <<'USAGE' -Usage: generate_ci_config.sh [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] +Usage: generate_ci_config.sh [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] [--shibboleth] USAGE exit 1 fi @@ -18,6 +18,7 @@ FLOWABLE=false S3COMPATSIGV4=false S3COMPATSIGV4_INST=false WIKI=false +SHIBBOLETH=false for arg in "$@"; do case "$arg" in @@ -42,6 +43,9 @@ for arg in "$@"; do --wiki) WIKI=true ;; + --shibboleth) + SHIBBOLETH=true + ;; *) echo "Unknown argument: ${arg}" >&2 exit 1 @@ -52,6 +56,28 @@ done cp "${BASE_CONFIG}" "${OUTPUT}" +if [[ "${SHIBBOLETH}" == "true" ]]; then + python3 - "${OUTPUT}" <<'PY' +import os +import sys +import yaml + +with open(sys.argv[1]) as source: + config = yaml.safe_load(source) +username = {'user': 'testuser1', 'admin': 'teststaff'}[os.environ['SHIBBOLETH_ROLE']] +config.update( + admin_rdm_url='http://192.168.168.167:18087/', + idp_name_1='RDM E2E Test IdP', idp_username_1=username, + idp_password_1='ci-fixture-password', + idp_name_2='RDM E2E Test IdP', idp_username_2='testuser2', + idp_password_2='ci-fixture-password', + institution_name='RDM E2E Institution', +) +with open(sys.argv[1], 'w') as output: + yaml.safe_dump(config, output, allow_unicode=True, sort_keys=False) +PY +fi + if [[ "${MINIO}" == "true" ]]; then if [[ -z "${S3COMPAT_ACCESS_KEY_1:-}" || -z "${S3COMPAT_SECRET_KEY_1:-}" ]]; then echo "S3 compat credentials are not set" >&2 diff --git a/.github/scripts/setup_rdm.sh b/.github/scripts/setup_rdm.sh index 2111510..211d337 100644 --- a/.github/scripts/setup_rdm.sh +++ b/.github/scripts/setup_rdm.sh @@ -83,6 +83,10 @@ start_rdm_services() { export SERVICES="mfr wb fakecas sharejs wb_worker worker web api ember_osf_web" fi + if [ "${SHIBBOLETH_ENABLED:-false}" = "true" ]; then + SERVICES="${SERVICES/fakecas/cas idp}" + fi + SERVICES="kaken_elasticsearch $SERVICES" if [ "${MINIO_ENABLED:-false}" = "true" ]; then @@ -98,6 +102,14 @@ start_rdm_services() { echo "Waiting for Ember build to complete..." local timeout="${TIMEOUT:-600}" # Default 10 minutes SERVICE="ember_osf_web" PATTERN="Build successful.*Serving on http://0.0.0.0:4200/" TIMEOUT="$timeout" check_service_logs + + if [ "${SHIBBOLETH_ENABLED:-false}" = "true" ]; then + SERVICE_NAME=IdP CHECK_COMMAND='docker-compose exec -T idp curl -fsS http://localhost:8080/idp/status' wait_for_service + python3 "${SCRIPT_DIR}/setup_shibboleth.py" idp-metadata "$PWD" + docker-compose up -d shibboleth_sp + SERVICE_NAME=SP CHECK_COMMAND='curl -fsS http://192.168.168.167:18088/Shibboleth.sso/Metadata' wait_for_service + SERVICE_NAME=CAS CHECK_COMMAND='curl -fsS http://localhost:8080/status' wait_for_service + fi } # Function to test RDM endpoints @@ -110,7 +122,11 @@ test_rdm_endpoints() { ENDPOINT_NAME="Ember OSF Web (port 4200)" ENDPOINT_URL="http://localhost:4200/" test_endpoint ENDPOINT_NAME="WaterButler (port 7777)" ENDPOINT_URL="http://192.168.168.167:7777/status" test_endpoint ENDPOINT_NAME="MFR (port 7778)" ENDPOINT_URL="http://localhost:7778/status" test_endpoint - ENDPOINT_NAME="FakeCAS (port 8080)" ENDPOINT_URL="http://localhost:8080/login" test_endpoint + if [ "${SHIBBOLETH_ENABLED:-false}" = "true" ]; then + ENDPOINT_NAME="CAS (port 8080)" ENDPOINT_URL="http://localhost:8080/status" test_endpoint + else + ENDPOINT_NAME="FakeCAS (port 8080)" ENDPOINT_URL="http://localhost:8080/login" test_endpoint + fi # Test admin endpoint if admin is enabled if [ "$include_admin" = "true" ]; then diff --git a/.github/scripts/setup_shibboleth.py b/.github/scripts/setup_shibboleth.py new file mode 100644 index 0000000..49acadb --- /dev/null +++ b/.github/scripts/setup_shibboleth.py @@ -0,0 +1,145 @@ +"""Configure a disposable RDM checkout after setup_rdm.sh's base configuration.""" +import argparse +import json +from pathlib import Path +import re +import shutil +import subprocess +import xml.etree.ElementTree as ET + +import yaml + +ROOT = Path(__file__).resolve().parents[2] / 'docker/shibboleth' +HOST = 'http://192.168.168.167' +IDP = f'{HOST}:18089/idp/shibboleth' +SP = f'{HOST}:18088/shibboleth' +MD = 'urn:oasis:names:tc:SAML:2.0:metadata' +DS = 'http://www.w3.org/2000/09/xmldsig#' +SAML = 'urn:oasis:names:tc:SAML:2.0:protocol' +ET.register_namespace('md', MD) +ET.register_namespace('ds', DS) + + +def metadata(path, entity, role, certificate): + root = ET.Element(f'{{{MD}}}EntityDescriptor', entityID=entity) + descriptor = ET.SubElement(root, f'{{{MD}}}{role}', protocolSupportEnumeration=SAML) + key = ET.SubElement(descriptor, f'{{{MD}}}KeyDescriptor') + info = ET.SubElement(key, f'{{{DS}}}KeyInfo') + data = ET.SubElement(info, f'{{{DS}}}X509Data') + ET.SubElement(data, f'{{{DS}}}X509Certificate').text = ''.join( + line for line in certificate.read_text().splitlines() if not line.startswith('-----')) + if role == 'SPSSODescriptor': + for index, port in enumerate((18088, 18087)): + ET.SubElement(descriptor, f'{{{MD}}}AssertionConsumerService', + Binding=f'{SAML.rsplit(":", 1)[0]}:bindings:HTTP-POST', + Location=f'{HOST}:{port}/Shibboleth.sso/SAML2/POST', index=str(index)) + else: + extensions = ET.Element(f'{{{MD}}}Extensions') + ET.SubElement(extensions, '{urn:mace:shibboleth:metadata:1.0}Scope', regexp='false').text = 'e2e.test' + descriptor.insert(0, extensions) + ET.SubElement(descriptor, f'{{{MD}}}SingleSignOnService', + Binding='urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect', + Location=f'{HOST}:18089/idp/profile/SAML2/Redirect/SSO') + ET.ElementTree(root).write(path, encoding='utf-8', xml_declaration=True) + + +def ds_config(port, target): + return '\n'.join(f'var {key} = {json.dumps(value)};' for key, value in { + 'wayf_URL': 'https://ds.gakunin.nii.ac.jp/WAYF', + 'wayf_sp_entityID': SP, + 'wayf_sp_handlerURL': f'{HOST}:{port}/Shibboleth.sso', + 'wayf_sp_samlDSURL': f'{HOST}:{port}/Shibboleth.sso/Login', + 'wayf_return_url': target, + 'wayf_auto_login': False, + 'wayf_additional_idps': [{'name': 'RDM E2E Test IdP', 'entityID': IDP}], + }.items()) + '\n' + + +def configure(rdm, cas_image): + for directory, suffix in (('shields', ''), ('shields-rounded-corners', '-rounded-corners')): + images = rdm / 'website/static/img/institutions' / directory + shutil.copyfile(images / f'gakunin-shield{suffix}.png', images / f'e2esh-shield{suffix}.png') + runtime = rdm / 'shibboleth' + runtime.mkdir() + exchange = runtime / 'exchange' + exchange.mkdir() + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:3072', '-nodes', + '-days', '2', '-subj', '/CN=RDM E2E SP', + '-keyout', str(exchange / 'sp-key.pem'), + '-out', str(exchange / 'sp-cert.pem')], check=True) + # The upstream SP drops privileges to shibd, while this fixture is mounted read-only. + (exchange / 'sp-key.pem').chmod(0o644) + metadata(exchange / 'sp-metadata.xml', SP, 'SPSSODescriptor', exchange / 'sp-cert.pem') + shutil.copytree(ROOT / 'sp', runtime / 'sp') + (runtime / 'sp/discovery/config.js').write_text(ds_config(18088, '')) + js = runtime / 'sp' / 'js' + js.mkdir() + (js / 'embedded-wayf_disp.js').write_text('') + (js / 'embedded-wayf_config.js').write_text(ds_config(18087, f'{HOST}:18087/account/shib-login')) + (runtime / 'dsconfig_js').write_text(ds_config(18088, f'{HOST}:18088/login?service={HOST}:5000/')) + shutil.copytree(ROOT / 'cas', runtime / 'cas') + properties = (rdm / 'etc/cas-overlay/cas.properties').read_text() + updates = { + 'server.name': f'{HOST}:18088', + 'osf.url': f'{HOST}:5000/', + 'osf.api.institutions.auth.url': 'http://api:8000/v2/institutions/auth/', + 'osf.api.institutions.auth.xslLocation': 'file:/etc/cas/institutions-auth.xsl', + 'service.registry.config.location': 'file:/etc/cas/services', + 'log4j.config.location': 'file:/etc/cas/log4j2.xml', + } + for key, value in updates.items(): + properties, count = re.subn(r'^' + re.escape(key) + r'=.*$', key + '=' + value, + properties, flags=re.MULTILINE) + assert count == 1, (key, count) + properties += '\nosf.api.institutions.login.availability.url=http://api:8000/v2/institutions/login_availability/\n' + (runtime / 'cas/cas.properties').write_text(properties) + for settings, content in { + 'website/settings/local.py': '\nUSE_EMBEDDED_DS = True\nLOGIN_BY_EPPN = True\n', + 'api/base/settings/local.py': "\nLOGIN_BY_EPPN = True\nALLOWED_HOSTS.append('api')\n", + 'admin/base/settings/local.py': ( + '\nENABLE_LOGIN_FORM = False\nENABLE_SHB_LOGIN = True\n' + 'EMBEDDED_DS_URL = "https://ds.gakunin.nii.ac.jp/WAYF/embedded-wayf.js"\n'), + }.items(): + with (rdm / settings).open('a') as output: + output.write(content) + env = rdm / '.docker-compose.env' + env.write_text(re.sub(r'^CAS_SERVER_URL=.*$', f'CAS_SERVER_URL={HOST}:18088', + env.read_text(), flags=re.MULTILINE)) + override = rdm / 'docker-compose.override.yml' + compose = yaml.safe_load(override.read_text()) + services = compose['services'] + services['cas']['volumes'] = [ + f'{runtime}/cas/cas.properties:/etc/cas/cas.properties:ro', + f'{runtime}/cas/institutions-auth.xsl:/etc/cas/institutions-auth.xsl:ro', + ] + services['cas']['image'] = cas_image + services['ember_osf_web']['volumes'] = [f'{runtime}/dsconfig_js:/code/public/dsconfig_js:ro'] + services['ember_osf_web']['environment'].update({ + 'NAV_EMBEDDEDDS_ENABLED': 'true', 'CAS_URL': f'{HOST}:18088', + }) + services['idp'] = { + 'image': 'rdm-idp:shibboleth', 'ports': ['18089:8080'], + 'volumes': [f'{exchange}:/exchange'], + } + services['shibboleth_sp'] = { + 'image': 'rdm-sp:shibboleth', 'ports': ['18088:80', '18087:81'], + 'volumes': [f'{exchange}:/exchange:ro', f'{runtime}/sp:/etc/e2e:ro', + f'{runtime}/sp/httpd.conf:/etc/httpd/conf.d/e2e.conf:ro', + f'{runtime}/sp/shibboleth2.xml:/etc/shibboleth/shibboleth2.xml:ro', + f'{runtime}/sp/attribute-map.xml:/etc/shibboleth/attribute-map.xml:ro'], + } + override.write_text(yaml.safe_dump(compose, sort_keys=False)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['configure', 'idp-metadata']) + parser.add_argument('rdm', type=Path) + parser.add_argument('--cas-image', default='rdm-cas:shibboleth') + args = parser.parse_args() + rdm = args.rdm.resolve() + if args.action == 'configure': + configure(rdm, args.cas_image) + else: + exchange = rdm / 'shibboleth/exchange' + metadata(exchange / 'idp-metadata.xml', IDP, 'IDPSSODescriptor', exchange / 'idp-signing.crt') diff --git a/.github/scripts/setup_test_data.py b/.github/scripts/setup_test_data.py index aae1f7d..582f263 100644 --- a/.github/scripts/setup_test_data.py +++ b/.github/scripts/setup_test_data.py @@ -4,6 +4,7 @@ WIKI_ENABLED = os.environ.get('WIKI_ENABLED', 'false').lower() == 'true' MAPCORE_GROUP_ENABLED = os.environ.get('MAPCORE_GROUP_ENABLED', 'false').lower() == 'true' +SHIBBOLETH_ENABLED = os.environ.get('SHIBBOLETH_ENABLED', 'false').lower() == 'true' INSTITUTION_NAME = 'Virginia Tech [Test]' @@ -81,6 +82,20 @@ }, ] +if SHIBBOLETH_ENABLED: + INSTITUTION_NAME = 'RDM E2E Institution' + Institution.objects.create( + _id='e2esh', name=INSTITUTION_NAME, domains=['e2e.test'], + logo_name='e2esh-shield.png', + email_domains=['example.com'], delegation_protocol='saml-shib', + login_url='http://192.168.168.167:18088/Shibboleth.sso/Login', + ) + test_users = test_users[:3] + for user_data in test_users: + # Staff privileges must come from the IdP entitlement at login. + user_data['is_superuser'] = False + user_data['is_staff'] = False + def create_rdmuserkey(user): # このスクリプトは `manage.py shell < file` で exec されるため、 # 関数内で使うシンボルは関数内で import する（トップレベル import は関数から見えない） @@ -132,6 +147,8 @@ def create_rdmuserkey(user): date_registered=timezone.now() ) user.set_password(user_data['password']) + if SHIBBOLETH_ENABLED: + user.eppn = username.split('@')[0] + '@e2e.test' user.save() # Set additional fields after save user.is_registered = True @@ -165,8 +182,8 @@ def create_rdmuserkey(user): user.emails.create(address=username) print(f"Created test user: {username}") - # Create user key (only needed when Wiki is enabled) - if WIKI_ENABLED: + # Federated login checks for an existing user key before allowing access. + if WIKI_ENABLED or SHIBBOLETH_ENABLED: create_rdmuserkey(user) # Skip project creation for dedicated CI users (e.g. project limit test user) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index bcf5dff..37f62ea 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -32,9 +32,25 @@ on: required: false default: '' cas_image: - description: 'Custom CAS image (default: niicloudoperation/rdm-cas-overlay:latest)' + description: 'Real CAS image; overrides CAS source settings for Shibboleth tests (not FakeCAS)' required: false default: '' + cas_repository: + description: 'CAS source repository for Shibboleth login tests' + required: false + default: 'RCOSDP/RDM-cas-overlay' + cas_branch: + description: 'CAS source ref for Shibboleth login tests' + required: false + default: 'develop' + sp_repository: + description: 'Shibboleth SP source repository' + required: false + default: 'RCOSDP/RDM-Shibboleth-SP' + sp_branch: + description: 'Shibboleth SP source ref' + required: false + default: 'main' mfr_image: description: 'Custom MFR image (default: niicloudoperation/rdm-modular-file-renderer:latest)' required: false @@ -109,6 +125,7 @@ jobs: runs-on: ubuntu-latest outputs: wb_custom: ${{ steps.wb_params.outputs.branch != '' }} + shibboleth_cas_image: ${{ steps.shibboleth_params.outputs.cas_image || 'rdm-cas:shibboleth' }} steps: - name: Checkout test repository uses: actions/checkout@v4 @@ -168,6 +185,79 @@ jobs: cd .. docker save -o prebuilt-images/rdm-waterbutler-custom.tar rdm-waterbutler:custom + - name: Parse Shibboleth source parameters + id: shibboleth_params + run: | + python3 - <<'PY' + import json + import os + import re + + with open(os.environ['GITHUB_EVENT_PATH']) as source: + event = json.load(source) + if os.environ['GITHUB_EVENT_NAME'] == 'pull_request': + parameters = dict(re.findall( + r'^-[ \t]*(CAS_IMAGE|CAS_REPOSITORY|CAS_BRANCH|SP_REPOSITORY|SP_BRANCH):[ \t]*([^\r\n]*)', + event['pull_request']['body'] or '', re.MULTILINE)) + elif os.environ['GITHUB_EVENT_NAME'] == 'workflow_dispatch': + parameters = {key.upper(): value for key, value in event['inputs'].items()} + else: + parameters = {} + defaults = { + 'CAS_IMAGE': '', + 'CAS_REPOSITORY': 'RCOSDP/RDM-cas-overlay', + 'CAS_BRANCH': 'develop', + 'SP_REPOSITORY': 'RCOSDP/RDM-Shibboleth-SP', + 'SP_BRANCH': 'main', + } + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + for key, default in defaults.items(): + value = parameters.get(key, '').strip() or default + print(f'{key.lower()}={value}', file=output) + PY + + - name: Checkout CAS source + if: steps.shibboleth_params.outputs.cas_image == '' + uses: actions/checkout@v4 + with: + repository: ${{ steps.shibboleth_params.outputs.cas_repository }} + ref: ${{ steps.shibboleth_params.outputs.cas_branch }} + path: RDM-cas-overlay + + - name: Checkout Shibboleth SP source + uses: actions/checkout@v4 + with: + repository: ${{ steps.shibboleth_params.outputs.sp_repository }} + ref: ${{ steps.shibboleth_params.outputs.sp_branch }} + path: RDM-Shibboleth-SP + + - name: Build Shibboleth authentication images + env: + CAS_IMAGE: ${{ steps.shibboleth_params.outputs.cas_image }} + CAS_REPOSITORY: ${{ steps.shibboleth_params.outputs.cas_repository }} + SP_REPOSITORY: ${{ steps.shibboleth_params.outputs.sp_repository }} + run: | + mkdir -p shibboleth-images + images=(rdm-sp:shibboleth rdm-idp:shibboleth) + if [[ -n "$CAS_IMAGE" ]]; then + printf 'CAS_IMAGE=%s\n' "$CAS_IMAGE" > shibboleth-images/revisions.txt + else + docker build --target dist -t rdm-cas:shibboleth RDM-cas-overlay + images+=(rdm-cas:shibboleth) + printf 'CAS_SOURCE=%s@%s\n' "$CAS_REPOSITORY" "$(git -C RDM-cas-overlay rev-parse HEAD)" > shibboleth-images/revisions.txt + fi + docker build -t rdm-sp:shibboleth RDM-Shibboleth-SP + docker build -t rdm-idp:shibboleth e2e-tests/docker/shibboleth/idp + docker save -o shibboleth-images/authentication.tar "${images[@]}" + printf 'SP_SOURCE=%s@%s\n' "$SP_REPOSITORY" "$(git -C RDM-Shibboleth-SP rev-parse HEAD)" >> shibboleth-images/revisions.txt + + - name: Upload Shibboleth authentication images + uses: actions/upload-artifact@v4 + with: + name: shibboleth-images + path: shibboleth-images/ + retention-days: 7 + - name: Upload prebuilt images uses: actions/upload-artifact@v4 with: @@ -238,6 +328,44 @@ jobs: テスト手順-管理者機能-アナウンス.ipynb テスト手順-管理者機能-証跡管理.ipynb テスト手順-管理者機能-NIIストレージのクォータ.ipynb + - name: user-shibboleth + display_name: "User Login (Shibboleth)" + shibboleth_enabled: true + shibboleth_role: user + include_admin: true + skip_admin: true + skip_login: false + skip_metadata: true + skip_default_storage: true + skip_autofill: true + exclude_notebooks: | + テスト手順-未ログイン.ipynb + - name: admin-shibboleth + display_name: "Admin Login (Shibboleth)" + shibboleth_enabled: true + shibboleth_role: admin + include_admin: true + skip_admin: false + skip_login: true + skip_metadata: true + skip_default_storage: true + skip_autofill: true + exclude_notebooks: | + テスト手順-管理者機能-未ログイン.ipynb + テスト手順-管理者機能-ログイン可否設定.ipynb + テスト手順-管理者機能-ノード管理.ipynb + テスト手順-管理者機能-ユーザ管理.ipynb + テスト手順-管理者機能-機関設定.ipynb + テスト手順-管理者機能-RDMユーザメール.ipynb + テスト手順-管理者機能-RDM登録.ipynb + テスト手順-管理者機能-メンテナンスアラート.ipynb + テスト手順-管理者機能-アドオン利用制御.ipynb + テスト手順-管理者機能-利用統計.ipynb + テスト手順-管理者機能-アナウンス.ipynb + テスト手順-管理者機能-証跡管理.ipynb + テスト手順-管理者機能-NIIストレージのクォータ.ipynb + テスト手順-管理者機能-機関ストレージのクォータ.ipynb + 取りまとめ-管理者機能-プロジェクト制限数.ipynb - name: migration-from-20250906 display_name: "Migration Test from 20250906" include_admin: true @@ -475,6 +603,8 @@ jobs: mapcore_group_enabled: true name: E2E ${{ matrix.test-group.display_name }} env: + SHIBBOLETH_ENABLED: ${{ matrix.test-group.shibboleth_enabled == true && 'true' || 'false' }} + SHIBBOLETH_ROLE: ${{ matrix.test-group.shibboleth_role }} MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }} @@ -726,6 +856,21 @@ jobs: fi rm -rf prebuilt-images + - name: Download Shibboleth authentication images + if: env.SHIBBOLETH_ENABLED == 'true' + uses: actions/download-artifact@v4 + with: + name: shibboleth-images + path: shibboleth-images + + - name: Load Shibboleth authentication images + if: env.SHIBBOLETH_ENABLED == 'true' + run: | + docker load -i shibboleth-images/authentication.tar + mkdir -p e2e-tests/result + mv shibboleth-images/revisions.txt e2e-tests/result/authentication-revisions.txt + rm -r shibboleth-images + - name: Install Docker Compose run: | sudo curl -L "https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose @@ -1065,6 +1210,13 @@ jobs: source ${{ github.workspace }}/e2e-tests/.github/scripts/setup_rdm.sh create_docker_override + - name: Configure Shibboleth authentication + if: env.SHIBBOLETH_ENABLED == 'true' + working-directory: RDM-osf.io + env: + SHIBBOLETH_CAS_IMAGE: ${{ needs.build-images.outputs.shibboleth_cas_image }} + run: python3 ../e2e-tests/.github/scripts/setup_shibboleth.py configure . --cas-image "$SHIBBOLETH_CAS_IMAGE" + - name: Install RDM requirements working-directory: RDM-osf.io run: | @@ -1361,7 +1513,7 @@ jobs: docker cp ${{ github.workspace }}/e2e-tests/.github/scripts/setup_test_data.py "$(docker-compose ps -q web)":/tmp/setup_test_data.py # Execute the script and capture project IDs - docker-compose exec -T -e WIKI_ENABLED="${WIKI_ENABLED}" -e MAPCORE_GROUP_ENABLED="${MAPCORE_GROUP_ENABLED}" web bash -c "python3 manage.py shell < /tmp/setup_test_data.py" | tee /tmp/setup_output.txt + docker-compose exec -T -e WIKI_ENABLED="${WIKI_ENABLED}" -e MAPCORE_GROUP_ENABLED="${MAPCORE_GROUP_ENABLED}" -e SHIBBOLETH_ENABLED="${SHIBBOLETH_ENABLED}" web bash -c "python3 manage.py shell < /tmp/setup_test_data.py" | tee /tmp/setup_output.txt # Extract project IDs and names from output PROJECT_ID_1=$(grep "PROJECT_ID_testuser1@example.com:" /tmp/setup_output.txt | cut -d' ' -f2) @@ -1481,6 +1633,9 @@ jobs: EOF args=(ci.config.yaml base_ci.config.yaml) + if [ "${SHIBBOLETH_ENABLED}" = "true" ]; then + args+=(--shibboleth) + fi if [ "${MINIO_ENABLED}" = "true" ]; then args+=(--minio) fi @@ -1499,7 +1654,7 @@ jobs: if [ "${S3COMPATSIGV4_INST_ENABLED}" = "true" ]; then args+=(--s3compatsigv4-inst) fi - if [ "${WIKI_ENABLED}" = "true" ]; then + if [ "${WIKI_ENABLED}" = "true" ] && [ "${SHIBBOLETH_ENABLED}" != "true" ]; then args+=(--wiki) fi diff --git a/docker/shibboleth/cas/institutions-auth.xsl b/docker/shibboleth/cas/institutions-auth.xsl new file mode 100644 index 0000000..61cf5e7 --- /dev/null +++ b/docker/shibboleth/cas/institutions-auth.xsl @@ -0,0 +1,20 @@ + + + + Unexpected authentication provider + + + e2esh + + + + + + + + + + + + + diff --git a/docker/shibboleth/idp/Dockerfile b/docker/shibboleth/idp/Dockerfile new file mode 100644 index 0000000..c16e9c9 --- /dev/null +++ b/docker/shibboleth/idp/Dockerfile @@ -0,0 +1,22 @@ +FROM tomcat:11-jdk17-temurin + +ARG IDP_VERSION=5.2.3 +RUN curl -fsSL "https://shibboleth.net/downloads/identity-provider/${IDP_VERSION}/shibboleth-identity-provider-${IDP_VERSION}.tar.gz" -o /tmp/idp.tar.gz \ + && mkdir /opt/idp-distribution \ + && tar -xzf /tmp/idp.tar.gz --strip-components=1 -C /opt/idp-distribution \ + && rm /tmp/idp.tar.gz +# This isolated HTTP fixture cannot use the distribution's __Host- secure cookie. +RUN sed -i -e 's/__Host-JSESSIONID/JSESSIONID_IDP/' \ + -e 's|/|/idp|' \ + -e 's|true|false|' \ + /opt/idp-distribution/webapp/WEB-INF/web.xml +COPY consent.css /opt/consent.css +COPY install.properties /opt/install.properties +RUN curl -fsSL https://repo.maven.apache.org/maven2/jakarta/servlet/jsp/jstl/jakarta.servlet.jsp.jstl-api/3.0.2/jakarta.servlet.jsp.jstl-api-3.0.2.jar -o /usr/local/tomcat/lib/jakarta.servlet.jsp.jstl-api.jar \ + && curl -fsSL https://repo.maven.apache.org/maven2/org/glassfish/web/jakarta.servlet.jsp.jstl/3.0.1/jakarta.servlet.jsp.jstl-3.0.1.jar -o /usr/local/tomcat/lib/jakarta.servlet.jsp.jstl.jar +COPY conf /opt/e2e-conf +COPY start.sh /usr/local/bin/start-idp +RUN mkdir -p /usr/local/tomcat/conf/Catalina/localhost \ + && echo '' > /usr/local/tomcat/conf/Catalina/localhost/idp.xml +ENV CATALINA_OPTS="-Didp.home=/opt/shibboleth-idp -Xmx512m" +CMD ["bash", "/usr/local/bin/start-idp"] diff --git a/docker/shibboleth/idp/conf/attribute-filter.xml b/docker/shibboleth/idp/conf/attribute-filter.xml new file mode 100644 index 0000000..38416d9 --- /dev/null +++ b/docker/shibboleth/idp/conf/attribute-filter.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + diff --git a/docker/shibboleth/idp/conf/attribute-resolver.xml b/docker/shibboleth/idp/conf/attribute-resolver.xml new file mode 100644 index 0000000..2c779b7 --- /dev/null +++ b/docker/shibboleth/idp/conf/attribute-resolver.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + Test User 1testuser1 + Test Staffteststaff + Test User 2testuser2 + + + + urn:rdm:e2e:membertestuser1testuser2 + GakuNinRDMAdminteststaff + + + E2E + User + RDM E2E Institution + member + + diff --git a/docker/shibboleth/idp/conf/authn/password-authn-config.xml b/docker/shibboleth/idp/conf/authn/password-authn-config.xml new file mode 100644 index 0000000..cb3918e --- /dev/null +++ b/docker/shibboleth/idp/conf/authn/password-authn-config.xml @@ -0,0 +1,9 @@ + + + + + diff --git a/docker/shibboleth/idp/conf/metadata-providers.xml b/docker/shibboleth/idp/conf/metadata-providers.xml new file mode 100644 index 0000000..69c028c --- /dev/null +++ b/docker/shibboleth/idp/conf/metadata-providers.xml @@ -0,0 +1,4 @@ + diff --git a/docker/shibboleth/idp/conf/relying-party.xml b/docker/shibboleth/idp/conf/relying-party.xml new file mode 100644 index 0000000..3a5bb24 --- /dev/null +++ b/docker/shibboleth/idp/conf/relying-party.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + diff --git a/docker/shibboleth/idp/consent.css b/docker/shibboleth/idp/consent.css new file mode 100644 index 0000000..8795a01 --- /dev/null +++ b/docker/shibboleth/idp/consent.css @@ -0,0 +1,16 @@ +/* Existing login procedures operate the native consent radio buttons. */ +input[name="_shib_idp_consentOptions"] { + position: static; + opacity: 1; + height: auto; + width: auto; +} + +input[name="_shib_idp_consentOptions"] + label { + padding-left: 0; +} + +input[name="_shib_idp_consentOptions"] + label::before, +input[name="_shib_idp_consentOptions"] + label::after { + content: none; +} diff --git a/docker/shibboleth/idp/install.properties b/docker/shibboleth/idp/install.properties new file mode 100644 index 0000000..90ea5a8 --- /dev/null +++ b/docker/shibboleth/idp/install.properties @@ -0,0 +1 @@ +idp.scope=e2e.test diff --git a/docker/shibboleth/idp/start.sh b/docker/shibboleth/idp/start.sh new file mode 100644 index 0000000..bab71d6 --- /dev/null +++ b/docker/shibboleth/idp/start.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -euo pipefail + +# Generate signing and encryption keys for this disposable run, never in the image. +if [ ! -f /opt/shibboleth-idp/war/idp.war ]; then + /opt/idp-distribution/bin/install.sh --noPrompt \ + --targetDir /opt/shibboleth-idp --hostName idp.e2e.test \ + --entityID http://192.168.168.167:18089/idp/shibboleth \ + --propertyFile /opt/install.properties \ + --keystorePassword ci-fixture-only --sealerPassword ci-fixture-only + /opt/shibboleth-idp/bin/module.sh -e idp.authn.Password + /opt/shibboleth-idp/bin/module.sh -e idp.intercept.Consent + cat /opt/consent.css >> /opt/shibboleth-idp/edit-webapp/css/placeholder.css + /opt/shibboleth-idp/bin/build.sh --noPrompt +fi +cp -R /opt/e2e-conf/. /opt/shibboleth-idp/conf/ +cat >> /opt/shibboleth-idp/conf/idp.properties <<'EOF' +idp.cookie.secure = false +idp.session.cookieName = shib_idp_session +idp.session.StorageService = shibboleth.StorageService +EOF +for user in testuser1 testuser2 teststaff; do + printf '%s:%s\n' "$user" "$(openssl passwd -6 ci-fixture-password)" +done > /opt/shibboleth-idp/conf/authn/htpasswd.txt +cp /opt/shibboleth-idp/credentials/idp-signing.crt /exchange/idp-signing.crt +chmod 644 /exchange/idp-signing.crt +exec catalina.sh run diff --git a/docker/shibboleth/sp/attribute-map.xml b/docker/shibboleth/sp/attribute-map.xml new file mode 100644 index 0000000..c753d69 --- /dev/null +++ b/docker/shibboleth/sp/attribute-map.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/docker/shibboleth/sp/discovery/index.html b/docker/shibboleth/sp/discovery/index.html new file mode 100644 index 0000000..9a899e8 --- /dev/null +++ b/docker/shibboleth/sp/discovery/index.html @@ -0,0 +1,18 @@ + + + + + RDM E2E - 所属機関の選択 + + +

所属機関の選択

+ + + + + diff --git a/docker/shibboleth/sp/headers.conf b/docker/shibboleth/sp/headers.conf new file mode 100644 index 0000000..7f7a991 --- /dev/null +++ b/docker/shibboleth/sp/headers.conf @@ -0,0 +1,21 @@ +# Never accept caller-supplied identity headers at the proxy boundary. +RequestHeader unset AUTH-eppn +RequestHeader unset AUTH-entitlement +RequestHeader unset AUTH-displayName +RequestHeader unset AUTH-mail +RequestHeader unset AUTH-givenName +RequestHeader unset AUTH-sn +RequestHeader unset AUTH-o +RequestHeader unset AUTH-Shib-Session-ID +RequestHeader unset AUTH-Shib-Identity-Provider +RequestHeader unset REMOTE_USER +RequestHeader set AUTH-eppn "%{eppn}e" env=eppn +RequestHeader set AUTH-entitlement "%{entitlement}e" env=entitlement +RequestHeader set AUTH-displayName "%{displayName}e" env=displayName +RequestHeader set AUTH-mail "%{mail}e" env=mail +RequestHeader set AUTH-givenName "%{givenName}e" env=givenName +RequestHeader set AUTH-sn "%{sn}e" env=sn +RequestHeader set AUTH-o "%{o}e" env=o +RequestHeader set AUTH-Shib-Session-ID "%{Shib-Session-ID}e" env=Shib-Session-ID +RequestHeader set AUTH-Shib-Identity-Provider "%{Shib-Identity-Provider}e" env=Shib-Identity-Provider +RequestHeader set REMOTE_USER "%{eppn}e" env=eppn diff --git a/docker/shibboleth/sp/httpd.conf b/docker/shibboleth/sp/httpd.conf new file mode 100644 index 0000000..10cfc37 --- /dev/null +++ b/docker/shibboleth/sp/httpd.conf @@ -0,0 +1,40 @@ +Listen 81 +ServerName 192.168.168.167 + + ServerName http://192.168.168.167:18088 + UseCanonicalName On + ProxyPreserveHost On + Include /etc/e2e/headers.conf + Alias /discovery/ /etc/e2e/discovery/ + + Require all granted + + ProxyPass /discovery/ ! + ProxyPass /Shibboleth.sso ! + ProxyPass / http://cas:8080/ + ProxyPassReverse / http://cas:8080/ + + AuthType shibboleth + ShibRequestSetting requireSession 1 + Require shib-session + + + + ServerName http://192.168.168.167:18087 + UseCanonicalName On + ProxyPreserveHost On + Include /etc/e2e/headers.conf + Alias /js/ /etc/e2e/js/ + + Require all granted + + ProxyPass /js/ ! + ProxyPass /Shibboleth.sso ! + ProxyPass / http://admin:8001/ + ProxyPassReverse / http://admin:8001/ + + AuthType shibboleth + ShibRequestSetting requireSession 1 + Require shib-session + + diff --git a/docker/shibboleth/sp/shibboleth2.xml b/docker/shibboleth/sp/shibboleth2.xml new file mode 100644 index 0000000..01c7c81 --- /dev/null +++ b/docker/shibboleth/sp/shibboleth2.xml @@ -0,0 +1,16 @@ + + + + SAML2 + SAML2 Local + + + + + + + + + + +