From 4de97183aee058b2080a6dd391554af2ae7f1f49 Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 2 Sep 2026 17:02:56 -0700 Subject: [PATCH 1/2] feat(stalwart): add SSM Session Manager access to management nodes Attach AmazonSSMManagedInstanceCore to the shared Stalwart node IAM role and explicitly install/enable amazon-ssm-agent in the node user-data, so operators can reach the management dashboard (port 8080) via AWS-StartPortForwardingSession with no bastion and no SG changes. Document both the existing SSH tunnel and the new SSM method in README.md, including why AWS-StartPortForwardingSessionToRemoteHost does not work here (the management port's SG only allows the private load balancer as a source). Co-Authored-By: Claude Code --- README.md | 90 ++++++++++++++++++++++++ pulumi/stalwart/__init__.py | 7 ++ pulumi/stalwart/iam.py | 26 ++++++- pulumi/stalwart_instance_user_data.sh.j2 | 8 ++- 4 files changed, 128 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f2d5025..215727d 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,96 @@ Now you have access to the admin panel by pointing a browser on your local machi with which to run TLS. As a result, the admin panel will not load over HTTPS. You will have to disable TLS on this by manually editing the service configuration on the node. There are more details on how to manage nodes below. +This works because the tunnel forwards the *node's own* loopback `8080` back to your machine over a connection you +already own (the SSH session), rather than routing new network traffic in from outside. The SSM-based method below +uses the same trick over a different transport, which is why neither of them needs the bastion's or the private load +balancer's security group opened up for your IP. + + +### Accessing the Management Dashboard via AWS SSO / SSM (No Bastion, No SSH Keys) + +As an alternative to the SSH tunnel above, you can reach the management dashboard through +[AWS Systems Manager Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html), +authenticating with your AWS SSO login instead of an SSH key or a bastion host. + +**Why this works without opening any security group:** the AWS-managed `AWS-StartPortForwardingSession` document +runs entirely on the target instance. It forwards a port on the instance's own loopback interface back to your +machine over the SSM agent's outbound WebSocket connection to the Systems Manager service. That traffic never enters +the VPC as routed network traffic, so the instance's inbound security group rules are never evaluated. No SG changes, +no inbound rule for any IP, ever — same principle as the SSH tunnel above, just over a different transport. + +This is **not** the same as `AWS-StartPortForwardingSessionToRemoteHost`, which proxies real, routed network traffic +through a jump host to a *different* target. That document *is* still subject to the target's security group. If you +try to relay through the bastion with `ToRemoteHost` pointed at a management node's private IP on port 8080, it will +fail with `Connection to destination port failed`, because the management port's security group only allows the +private "management" load balancer as a source, not the bastion. Always target the management node itself with +plain `AWS-StartPortForwardingSession`, not the bastion. + +#### Prerequisites + +- Your IAM role/user must have `ssm:StartSession` permission (granted via your AWS SSO permission set), and the + target instance's IAM role must have the `AmazonSSMManagedInstanceCore` managed policy attached (this is built + into the Stalwart node role by this project; see `pulumi/stalwart/iam.py`). +- The instance needs outbound access to the SSM/EC2Messages/SSMMessages endpoints, either via the Internet or VPC + endpoints. Stalwart nodes already have this via their NAT/egress path. +- The `session-manager-plugin` must be installed locally. + + **Normal install (root/dnf/yum access):** + + ```bash + sudo yum install -y https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm + ``` + + **No-root / immutable-OS fallback** (for bootc-style systems where `dnf`/`rpm` can't install packages directly): + + ```bash + cd /tmp + curl -sO https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm + mkdir -p /tmp/smp-extract && cd /tmp/smp-extract + rpm2cpio ../session-manager-plugin.rpm | cpio -idmv + mkdir -p ~/.local/bin + cp /tmp/smp-extract/usr/local/sessionmanagerplugin/bin/session-manager-plugin ~/.local/bin/ + chmod +x ~/.local/bin/session-manager-plugin + ``` + + Make sure `~/.local/bin` is on your `PATH`, then verify the install: + + ```bash + session-manager-plugin + # The Session Manager plugin was installed successfully. + ``` + +#### Steps + +1. Confirm you're logged into the right AWS SSO identity: + + ```bash + aws sts get-caller-identity --profile + ``` + +2. Confirm the target node is registered and online in SSM: + + ```bash + aws ssm describe-instance-information \ + --profile --region \ + --filters "Key=InstanceIds,Values=" \ + --query 'InstanceInformationList[].{Id:InstanceId,Ping:PingStatus}' \ + --output table + ``` + +3. Start the port-forwarding session directly to the management node (no bastion involved): + + ```bash + aws ssm start-session \ + --profile --region \ + --target \ + --document-name AWS-StartPortForwardingSession \ + --parameters '{"portNumber":["8080"],"localPortNumber":["8080"]}' + ``` + +4. Browse to https://localhost:8080/. As with the SSH tunnel above, expect a self-signed certificate warning on a + freshly-bootstrapped cluster that hasn't yet been issued a real TLS certificate. + ### Bootstrapping a Stalwart Node diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index 7479fa4..efb655a 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -90,6 +90,10 @@ class StalwartCluster(tb_pulumi.ThunderbirdComponentResource): - *node_profile_s3_policy_attachment* - The `aws.iam.PolicyAttachment `_ resource between the policy granting access to the S3 bucket Stalwart uses for blob storage and the instance profile. + - *node_profile_ssm_policy_attachment* - The `aws.iam.RolePolicyAttachment + `_ resource attaching the + AWS-managed ``AmazonSSMManagedInstanceCore`` policy to the node role, so the SSM agent can register nodes + with Systems Manager and support ``AWS-StartPortForwardingSession`` port forwarding. - *node_sgs* - Dict of :py:class:`tb_pulumi.network.SecurityGroupWithRules` created for each node to support its enabled services, identified by their node_id. - *private_lbs* - Dict mapping service names to the :py:class:`StalwartLoadBalancer` s which expose those @@ -333,6 +337,7 @@ def __init__( profile_postboot_attachment, profile_s3_attachment, profile_logwrite_attachment, + profile_ssm_attachment, profile, ) = stalwart_iam.iam( self, @@ -347,6 +352,7 @@ def __init__( subnet = nodes[node_id].pop('subnet', None) or self.private_subnets[idx % len(self.private_subnets)] depends_on = [ profile, + profile_ssm_attachment, redis_secret, s3_secret, *self.private_load_balancer_security_groups.values(), @@ -428,6 +434,7 @@ def __init__( 'node_profile_logwrite_attachment': profile_logwrite_attachment, 'node_profile_postboot_policy_attachment': profile_postboot_attachment, 'node_profile_s3_policy_attachment': profile_s3_attachment, + 'node_profile_ssm_policy_attachment': profile_ssm_attachment, 'node_sgs': self.node_sgs, 'private_lbs': private_lbs, 'private_lb_dns': private_lb_dns, diff --git a/pulumi/stalwart/iam.py b/pulumi/stalwart/iam.py index c1fb6e2..bc6226a 100644 --- a/pulumi/stalwart/iam.py +++ b/pulumi/stalwart/iam.py @@ -8,12 +8,23 @@ from tb_pulumi.constants import ASSUME_ROLE_POLICY +#: Managed policy granting the permissions the SSM agent needs to register with Systems Manager and support +#: session-manager port forwarding (``AWS-StartPortForwardingSession``) without any inbound security group rule. +AMAZON_SSM_MANAGED_INSTANCE_CORE_ARN = 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore' + + def iam( self, log_group_arn: str, s3_policy: aws.iam.Policy, ) -> tuple[ - aws.iam.Policy, aws.iam.Role, aws.iam.RolePolicyAttachment, aws.iam.RolePolicyAttachment, aws.iam.InstanceProfile + aws.iam.Policy, + aws.iam.Role, + aws.iam.RolePolicyAttachment, + aws.iam.RolePolicyAttachment, + aws.iam.RolePolicyAttachment, + aws.iam.RolePolicyAttachment, + aws.iam.InstanceProfile, ]: """Build IAM resources needed by Stalwart. @@ -22,7 +33,7 @@ def iam( :return: Series of IAM resources for Stalwart. :rtype: tuple[ tb_pulumi.iam.UserWithAccessKey, aws.iam.Policy, aws.iam.Role, aws.iam.RolePolicyAttachment, - aws.iam.InstanceProfile ] + aws.iam.RolePolicyAttachment, aws.iam.RolePolicyAttachment, aws.iam.InstanceProfile ] """ # Build a policy which will grant the nodes access to their own configuration data @@ -75,6 +86,16 @@ def iam( policy_arn=log_group_arn, ) + # Grant the node role SSM's managed instance policy. This lets the SSM agent register the node with Systems + # Manager and lets operators open `AWS-StartPortForwardingSession` sessions to it directly, without a bastion + # or any inbound security group rule. There's one shared role per cluster (see the instance profile below), so + # this is applied cluster-wide rather than only to nodes running the "management" service. + profile_ssm_attachment = aws.iam.RolePolicyAttachment( + f'{self.name}-rpa-nodeprofile-ssm', + role=role.name, + policy_arn=AMAZON_SSM_MANAGED_INSTANCE_CORE_ARN, + ) + profile = aws.iam.InstanceProfile(f'{self.name}-ip-nodeprofile', name=f'{self.name}-nodeprofile', role=role.name) return ( @@ -83,5 +104,6 @@ def iam( profile_postboot_attachment, profile_s3_attachment, profile_logwrite_attachment, + profile_ssm_attachment, profile, ) diff --git a/pulumi/stalwart_instance_user_data.sh.j2 b/pulumi/stalwart_instance_user_data.sh.j2 index b65b6d4..6c84e2e 100644 --- a/pulumi/stalwart_instance_user_data.sh.j2 +++ b/pulumi/stalwart_instance_user_data.sh.j2 @@ -23,7 +23,13 @@ enabled=1' > /etc/yum.repos.d/fluent-bit.repo # Update system, install dependencies dnf update -y -dnf install -y bzip2 docker fluent-bit python3.12 +dnf install -y amazon-ssm-agent bzip2 docker fluent-bit python3.12 + +# Amazon Linux 2023 ships the SSM agent preinstalled, but we install/enable it explicitly rather than assume, since +# that isn't guaranteed across every AMI variant. This lets operators reach a node's management dashboard via +# `aws ssm start-session ... --document-name AWS-StartPortForwardingSession` with no bastion and no inbound security +# group rule (see the README for details). Requires the node's IAM role to carry AmazonSSMManagedInstanceCore. +systemctl enable amazon-ssm-agent --now # Delete the default fluent-bit config; we'll template a new one in Phase 2 rm -f /etc/fluent-bit/fluent-bit.conf From a8ab40347350a0b2eb9bb928031885f96fcf6e52 Mon Sep 17 00:00:00 2001 From: Arron Atchison Date: Wed, 2 Sep 2026 17:12:56 -0700 Subject: [PATCH 2/2] fix(stalwart): drop explanatory comments, simplify plugin install docs Strip the inline/docstring comments added in the previous commit, keeping the code logic unchanged. Also replace the OS-specific session-manager-plugin install instructions in README.md with a link to AWS's own install docs, since the exact steps vary by OS. Co-Authored-By: Claude Code --- README.md | 24 +++--------------------- pulumi/stalwart/__init__.py | 4 ---- pulumi/stalwart/iam.py | 6 ------ pulumi/stalwart_instance_user_data.sh.j2 | 4 ---- 4 files changed, 3 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 215727d..27f082b 100644 --- a/README.md +++ b/README.md @@ -307,27 +307,9 @@ plain `AWS-StartPortForwardingSession`, not the bastion. into the Stalwart node role by this project; see `pulumi/stalwart/iam.py`). - The instance needs outbound access to the SSM/EC2Messages/SSMMessages endpoints, either via the Internet or VPC endpoints. Stalwart nodes already have this via their NAT/egress path. -- The `session-manager-plugin` must be installed locally. - - **Normal install (root/dnf/yum access):** - - ```bash - sudo yum install -y https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm - ``` - - **No-root / immutable-OS fallback** (for bootc-style systems where `dnf`/`rpm` can't install packages directly): - - ```bash - cd /tmp - curl -sO https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm - mkdir -p /tmp/smp-extract && cd /tmp/smp-extract - rpm2cpio ../session-manager-plugin.rpm | cpio -idmv - mkdir -p ~/.local/bin - cp /tmp/smp-extract/usr/local/sessionmanagerplugin/bin/session-manager-plugin ~/.local/bin/ - chmod +x ~/.local/bin/session-manager-plugin - ``` - - Make sure `~/.local/bin` is on your `PATH`, then verify the install: +- The `session-manager-plugin` must be installed locally. Follow AWS's + [Session Manager plugin installation instructions](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html) + for your OS, then verify the install: ```bash session-manager-plugin diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index efb655a..b80d70c 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -90,10 +90,6 @@ class StalwartCluster(tb_pulumi.ThunderbirdComponentResource): - *node_profile_s3_policy_attachment* - The `aws.iam.PolicyAttachment `_ resource between the policy granting access to the S3 bucket Stalwart uses for blob storage and the instance profile. - - *node_profile_ssm_policy_attachment* - The `aws.iam.RolePolicyAttachment - `_ resource attaching the - AWS-managed ``AmazonSSMManagedInstanceCore`` policy to the node role, so the SSM agent can register nodes - with Systems Manager and support ``AWS-StartPortForwardingSession`` port forwarding. - *node_sgs* - Dict of :py:class:`tb_pulumi.network.SecurityGroupWithRules` created for each node to support its enabled services, identified by their node_id. - *private_lbs* - Dict mapping service names to the :py:class:`StalwartLoadBalancer` s which expose those diff --git a/pulumi/stalwart/iam.py b/pulumi/stalwart/iam.py index bc6226a..a2d927e 100644 --- a/pulumi/stalwart/iam.py +++ b/pulumi/stalwart/iam.py @@ -8,8 +8,6 @@ from tb_pulumi.constants import ASSUME_ROLE_POLICY -#: Managed policy granting the permissions the SSM agent needs to register with Systems Manager and support -#: session-manager port forwarding (``AWS-StartPortForwardingSession``) without any inbound security group rule. AMAZON_SSM_MANAGED_INSTANCE_CORE_ARN = 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore' @@ -86,10 +84,6 @@ def iam( policy_arn=log_group_arn, ) - # Grant the node role SSM's managed instance policy. This lets the SSM agent register the node with Systems - # Manager and lets operators open `AWS-StartPortForwardingSession` sessions to it directly, without a bastion - # or any inbound security group rule. There's one shared role per cluster (see the instance profile below), so - # this is applied cluster-wide rather than only to nodes running the "management" service. profile_ssm_attachment = aws.iam.RolePolicyAttachment( f'{self.name}-rpa-nodeprofile-ssm', role=role.name, diff --git a/pulumi/stalwart_instance_user_data.sh.j2 b/pulumi/stalwart_instance_user_data.sh.j2 index 6c84e2e..6218006 100644 --- a/pulumi/stalwart_instance_user_data.sh.j2 +++ b/pulumi/stalwart_instance_user_data.sh.j2 @@ -25,10 +25,6 @@ enabled=1' > /etc/yum.repos.d/fluent-bit.repo dnf update -y dnf install -y amazon-ssm-agent bzip2 docker fluent-bit python3.12 -# Amazon Linux 2023 ships the SSM agent preinstalled, but we install/enable it explicitly rather than assume, since -# that isn't guaranteed across every AMI variant. This lets operators reach a node's management dashboard via -# `aws ssm start-session ... --document-name AWS-StartPortForwardingSession` with no bastion and no inbound security -# group rule (see the README for details). Requires the node's IAM role to carry AmazonSSMManagedInstanceCore. systemctl enable amazon-ssm-agent --now # Delete the default fluent-bit config; we'll template a new one in Phase 2