[WIP] Shibboleth認証による一般ユーザー・管理者のログイン試験をCIに追加

.github/pull_request_template.md

@@ -13,6 +13,12 @@
 - OSF_IMAGE: 
 - EMBER_IMAGE: 
 - CAS_IMAGE: 
+<!-- CAS_IMAGE specifies real CAS, not FakeCAS, and takes precedence over CAS source settings. -->
+- CAS_REPOSITORY:
+- CAS_BRANCH:
+- SP_REPOSITORY:
+- SP_BRANCH:
+<!-- CAS/SP source settings apply to Shibboleth login tests; blank values use RCOSDP/RDM-cas-overlay@develop and RCOSDP/RDM-Shibboleth-SP@main. -->
 - MFR_IMAGE: 
 - WB_IMAGE:
 - EXCLUDE_NOTEBOOKS: 

.github/scripts/generate_ci_config.sh

@@ -3,7 +3,7 @@ set -xeuo pipefail
 
 if [[ $# -lt 2 ]]; then
   cat >&2 <<'USAGE'
-Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki]
+Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] [--wiki] [--shibboleth]
 USAGE
   exit 1
 fi
@@ -18,6 +18,7 @@ FLOWABLE=false
 S3COMPATSIGV4=false
 S3COMPATSIGV4_INST=false
 WIKI=false
+SHIBBOLETH=false
 
 for arg in "$@"; do
   case "$arg" in
@@ -42,6 +43,9 @@ for arg in "$@"; do
     --wiki)
       WIKI=true
       ;;
+    --shibboleth)
+      SHIBBOLETH=true
+      ;;
     *)
       echo "Unknown argument: ${arg}" >&2
       exit 1
@@ -52,6 +56,28 @@ done
 
 cp "${BASE_CONFIG}" "${OUTPUT}"
 
+if [[ "${SHIBBOLETH}" == "true" ]]; then
+  python3 - "${OUTPUT}" <<'PY'
+import os
+import sys
+import yaml
+
+with open(sys.argv[1]) as source:
+    config = yaml.safe_load(source)
+username = {'user': 'testuser1', 'admin': 'teststaff'}[os.environ['SHIBBOLETH_ROLE']]
+config.update(
+    admin_rdm_url='http://192.168.168.167:18087/',
+    idp_name_1='RDM E2E Test IdP', idp_username_1=username,
+    idp_password_1='ci-fixture-password',
+    idp_name_2='RDM E2E Test IdP', idp_username_2='testuser2',
+    idp_password_2='ci-fixture-password',
+    institution_name='RDM E2E Institution',
+)
+with open(sys.argv[1], 'w') as output:
+    yaml.safe_dump(config, output, allow_unicode=True, sort_keys=False)
+PY
+fi
+
 if [[ "${MINIO}" == "true" ]]; then
   if [[ -z "${S3COMPAT_ACCESS_KEY_1:-}" || -z "${S3COMPAT_SECRET_KEY_1:-}" ]]; then
     echo "S3 compat credentials are not set" >&2

.github/scripts/setup_rdm.sh

@@ -83,6 +83,10 @@ start_rdm_services() {
         export SERVICES="mfr wb fakecas sharejs wb_worker worker web api ember_osf_web"
     fi
 
+    if [ "${SHIBBOLETH_ENABLED:-false}" = "true" ]; then
+        SERVICES="${SERVICES/fakecas/cas idp}"
+    fi
+
     SERVICES="kaken_elasticsearch $SERVICES"
 
     if [ "${MINIO_ENABLED:-false}" = "true" ]; then
@@ -98,6 +102,14 @@ start_rdm_services() {
     echo "Waiting for Ember build to complete..."
     local timeout="${TIMEOUT:-600}"  # Default 10 minutes
     SERVICE="ember_osf_web" PATTERN="Build successful.*Serving on http://0.0.0.0:4200/" TIMEOUT="$timeout" check_service_logs
+
+    if [ "${SHIBBOLETH_ENABLED:-false}" = "true" ]; then
+        SERVICE_NAME=IdP CHECK_COMMAND='docker-compose exec -T idp curl -fsS http://localhost:8080/idp/status' wait_for_service
+        python3 "${SCRIPT_DIR}/setup_shibboleth.py" idp-metadata "$PWD"
+        docker-compose up -d shibboleth_sp
+        SERVICE_NAME=SP CHECK_COMMAND='curl -fsS http://192.168.168.167:18088/Shibboleth.sso/Metadata' wait_for_service
+        SERVICE_NAME=CAS CHECK_COMMAND='curl -fsS http://localhost:8080/status' wait_for_service
+    fi
 }
 
 # Function to test RDM endpoints
@@ -110,7 +122,11 @@ test_rdm_endpoints() {
     ENDPOINT_NAME="Ember OSF Web (port 4200)" ENDPOINT_URL="http://localhost:4200/" test_endpoint
     ENDPOINT_NAME="WaterButler (port 7777)" ENDPOINT_URL="http://192.168.168.167:7777/status" test_endpoint
     ENDPOINT_NAME="MFR (port 7778)" ENDPOINT_URL="http://localhost:7778/status" test_endpoint
-    ENDPOINT_NAME="FakeCAS (port 8080)" ENDPOINT_URL="http://localhost:8080/login" test_endpoint
+    if [ "${SHIBBOLETH_ENABLED:-false}" = "true" ]; then
+        ENDPOINT_NAME="CAS (port 8080)" ENDPOINT_URL="http://localhost:8080/status" test_endpoint
+    else
+        ENDPOINT_NAME="FakeCAS (port 8080)" ENDPOINT_URL="http://localhost:8080/login" test_endpoint
+    fi
 
     # Test admin endpoint if admin is enabled
     if [ "$include_admin" = "true" ]; then

.github/scripts/setup_shibboleth.py

@@ -0,0 +1,145 @@
+"""Configure a disposable RDM checkout after setup_rdm.sh's base configuration."""
+import argparse
+import json
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import xml.etree.ElementTree as ET
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2] / 'docker/shibboleth'
+HOST = 'http://192.168.168.167'
+IDP = f'{HOST}:18089/idp/shibboleth'
+SP = f'{HOST}:18088/shibboleth'
+MD = 'urn:oasis:names:tc:SAML:2.0:metadata'
+DS = 'http://www.w3.org/2000/09/xmldsig#'
+SAML = 'urn:oasis:names:tc:SAML:2.0:protocol'
+ET.register_namespace('md', MD)
+ET.register_namespace('ds', DS)
+
+
+def metadata(path, entity, role, certificate):
+    root = ET.Element(f'{{{MD}}}EntityDescriptor', entityID=entity)
+    descriptor = ET.SubElement(root, f'{{{MD}}}{role}', protocolSupportEnumeration=SAML)
+    key = ET.SubElement(descriptor, f'{{{MD}}}KeyDescriptor')
+    info = ET.SubElement(key, f'{{{DS}}}KeyInfo')
+    data = ET.SubElement(info, f'{{{DS}}}X509Data')
+    ET.SubElement(data, f'{{{DS}}}X509Certificate').text = ''.join(
+        line for line in certificate.read_text().splitlines() if not line.startswith('-----'))
+    if role == 'SPSSODescriptor':
+        for index, port in enumerate((18088, 18087)):
+            ET.SubElement(descriptor, f'{{{MD}}}AssertionConsumerService',
+                          Binding=f'{SAML.rsplit(":", 1)[0]}:bindings:HTTP-POST',
+                          Location=f'{HOST}:{port}/Shibboleth.sso/SAML2/POST', index=str(index))
+    else:
+        extensions = ET.Element(f'{{{MD}}}Extensions')
+        ET.SubElement(extensions, '{urn:mace:shibboleth:metadata:1.0}Scope', regexp='false').text = 'e2e.test'
+        descriptor.insert(0, extensions)
+        ET.SubElement(descriptor, f'{{{MD}}}SingleSignOnService',
+                      Binding='urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect',
+                      Location=f'{HOST}:18089/idp/profile/SAML2/Redirect/SSO')
+    ET.ElementTree(root).write(path, encoding='utf-8', xml_declaration=True)
+
+
+def ds_config(port, target):
+    return '\n'.join(f'var {key} = {json.dumps(value)};' for key, value in {
+        'wayf_URL': 'https://ds.gakunin.nii.ac.jp/WAYF',
+        'wayf_sp_entityID': SP,
+        'wayf_sp_handlerURL': f'{HOST}:{port}/Shibboleth.sso',
+        'wayf_sp_samlDSURL': f'{HOST}:{port}/Shibboleth.sso/Login',
+        'wayf_return_url': target,
+        'wayf_auto_login': False,
+        'wayf_additional_idps': [{'name': 'RDM E2E Test IdP', 'entityID': IDP}],
+    }.items()) + '\n'
+
+
+def configure(rdm, cas_image):
+    for directory, suffix in (('shields', ''), ('shields-rounded-corners', '-rounded-corners')):
+        images = rdm / 'website/static/img/institutions' / directory
+        shutil.copyfile(images / f'gakunin-shield{suffix}.png', images / f'e2esh-shield{suffix}.png')
+    runtime = rdm / 'shibboleth'
+    runtime.mkdir()
+    exchange = runtime / 'exchange'
+    exchange.mkdir()
+    subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:3072', '-nodes',
+                    '-days', '2', '-subj', '/CN=RDM E2E SP',
+                    '-keyout', str(exchange / 'sp-key.pem'),
+                    '-out', str(exchange / 'sp-cert.pem')], check=True)
+    # The upstream SP drops privileges to shibd, while this fixture is mounted read-only.
+    (exchange / 'sp-key.pem').chmod(0o644)
+    metadata(exchange / 'sp-metadata.xml', SP, 'SPSSODescriptor', exchange / 'sp-cert.pem')
+    shutil.copytree(ROOT / 'sp', runtime / 'sp')
+    (runtime / 'sp/discovery/config.js').write_text(ds_config(18088, ''))
+    js = runtime / 'sp' / 'js'
+    js.mkdir()
+    (js / 'embedded-wayf_disp.js').write_text('')
+    (js / 'embedded-wayf_config.js').write_text(ds_config(18087, f'{HOST}:18087/account/shib-login'))
+    (runtime / 'dsconfig_js').write_text(ds_config(18088, f'{HOST}:18088/login?service={HOST}:5000/'))
+    shutil.copytree(ROOT / 'cas', runtime / 'cas')
+    properties = (rdm / 'etc/cas-overlay/cas.properties').read_text()
+    updates = {
+        'server.name': f'{HOST}:18088',
+        'osf.url': f'{HOST}:5000/',
+        'osf.api.institutions.auth.url': 'http://api:8000/v2/institutions/auth/',
+        'osf.api.institutions.auth.xslLocation': 'file:/etc/cas/institutions-auth.xsl',
+        'service.registry.config.location': 'file:/etc/cas/services',
+        'log4j.config.location': 'file:/etc/cas/log4j2.xml',
+    }
+    for key, value in updates.items():
+        properties, count = re.subn(r'^' + re.escape(key) + r'=.*$', key + '=' + value,
+                                    properties, flags=re.MULTILINE)
+        assert count == 1, (key, count)
+    properties += '\nosf.api.institutions.login.availability.url=http://api:8000/v2/institutions/login_availability/\n'
+    (runtime / 'cas/cas.properties').write_text(properties)
+    for settings, content in {
+        'website/settings/local.py': '\nUSE_EMBEDDED_DS = True\nLOGIN_BY_EPPN = True\n',
+        'api/base/settings/local.py': "\nLOGIN_BY_EPPN = True\nALLOWED_HOSTS.append('api')\n",
+        'admin/base/settings/local.py': (
+            '\nENABLE_LOGIN_FORM = False\nENABLE_SHB_LOGIN = True\n'
+            'EMBEDDED_DS_URL = "https://ds.gakunin.nii.ac.jp/WAYF/embedded-wayf.js"\n'),
+    }.items():
+        with (rdm / settings).open('a') as output:
+            output.write(content)
+    env = rdm / '.docker-compose.env'
+    env.write_text(re.sub(r'^CAS_SERVER_URL=.*$', f'CAS_SERVER_URL={HOST}:18088',
+                         env.read_text(), flags=re.MULTILINE))
+    override = rdm / 'docker-compose.override.yml'
+    compose = yaml.safe_load(override.read_text())
+    services = compose['services']
+    services['cas']['volumes'] = [
+        f'{runtime}/cas/cas.properties:/etc/cas/cas.properties:ro',
+        f'{runtime}/cas/institutions-auth.xsl:/etc/cas/institutions-auth.xsl:ro',
+    ]
+    services['cas']['image'] = cas_image
+    services['ember_osf_web']['volumes'] = [f'{runtime}/dsconfig_js:/code/public/dsconfig_js:ro']
+    services['ember_osf_web']['environment'].update({
+        'NAV_EMBEDDEDDS_ENABLED': 'true', 'CAS_URL': f'{HOST}:18088',
+    })
+    services['idp'] = {
+        'image': 'rdm-idp:shibboleth', 'ports': ['18089:8080'],
+        'volumes': [f'{exchange}:/exchange'],
+    }
+    services['shibboleth_sp'] = {
+        'image': 'rdm-sp:shibboleth', 'ports': ['18088:80', '18087:81'],
+        'volumes': [f'{exchange}:/exchange:ro', f'{runtime}/sp:/etc/e2e:ro',
+                    f'{runtime}/sp/httpd.conf:/etc/httpd/conf.d/e2e.conf:ro',
+                    f'{runtime}/sp/shibboleth2.xml:/etc/shibboleth/shibboleth2.xml:ro',
+                    f'{runtime}/sp/attribute-map.xml:/etc/shibboleth/attribute-map.xml:ro'],
+    }
+    override.write_text(yaml.safe_dump(compose, sort_keys=False))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('action', choices=['configure', 'idp-metadata'])
+    parser.add_argument('rdm', type=Path)
+    parser.add_argument('--cas-image', default='rdm-cas:shibboleth')
+    args = parser.parse_args()
+    rdm = args.rdm.resolve()
+    if args.action == 'configure':
+        configure(rdm, args.cas_image)
+    else:
+        exchange = rdm / 'shibboleth/exchange'
+        metadata(exchange / 'idp-metadata.xml', IDP, 'IDPSSODescriptor', exchange / 'idp-signing.crt')

.github/scripts/setup_test_data.py

@@ -4,6 +4,7 @@
 
 WIKI_ENABLED = os.environ.get('WIKI_ENABLED', 'false').lower() == 'true'
 MAPCORE_GROUP_ENABLED = os.environ.get('MAPCORE_GROUP_ENABLED', 'false').lower() == 'true'
+SHIBBOLETH_ENABLED = os.environ.get('SHIBBOLETH_ENABLED', 'false').lower() == 'true'
 
 INSTITUTION_NAME = 'Virginia Tech [Test]'
 
@@ -81,6 +82,20 @@
     },
 ]
 
+if SHIBBOLETH_ENABLED:
+    INSTITUTION_NAME = 'RDM E2E Institution'
+    Institution.objects.create(
+        _id='e2esh', name=INSTITUTION_NAME, domains=['e2e.test'],
+        logo_name='e2esh-shield.png',
+        email_domains=['example.com'], delegation_protocol='saml-shib',
+        login_url='http://192.168.168.167:18088/Shibboleth.sso/Login',
+    )
+    test_users = test_users[:3]
+    for user_data in test_users:
+        # Staff privileges must come from the IdP entitlement at login.
+        user_data['is_superuser'] = False
+        user_data['is_staff'] = False
+
 def create_rdmuserkey(user):
     # このスクリプトは `manage.py shell < file` で exec されるため、
     # 関数内で使うシンボルは関数内で import する（トップレベル import は関数から見えない）
@@ -132,6 +147,8 @@ def create_rdmuserkey(user):
             date_registered=timezone.now()
         )
         user.set_password(user_data['password'])
+        if SHIBBOLETH_ENABLED:
+            user.eppn = username.split('@')[0] + '@e2e.test'
         user.save()
         # Set additional fields after save
         user.is_registered = True
@@ -165,8 +182,8 @@ def create_rdmuserkey(user):
         user.emails.create(address=username)
         print(f"Created test user: {username}")
 
-        # Create user key (only needed when Wiki is enabled)
-        if WIKI_ENABLED:
+        # Federated login checks for an existing user key before allowing access.
+        if WIKI_ENABLED or SHIBBOLETH_ENABLED:
             create_rdmuserkey(user)
 
         # Skip project creation for dedicated CI users (e.g. project limit test user)