diff --git a/api/v1alpha/httpproxy_types.go b/api/v1alpha/httpproxy_types.go index 88ab9434..1ee46a91 100644 --- a/api/v1alpha/httpproxy_types.go +++ b/api/v1alpha/httpproxy_types.go @@ -444,8 +444,103 @@ type HostnameStatus struct { // +listMapKey=type // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` + + // DNSRecords lists every DNS record this hostname depends on, who publishes + // it, and whether it is in place. A record is Present only once it takes + // effect on the Internet. Records the user publishes stay listed while the + // hostname needs them, so this list alone says what is left to do. + // + // +listType=atomic + // +kubebuilder:validation:MaxItems=16 + // +optional + DNSRecords []HostnameDNSRecord `json:"dnsRecords,omitempty"` +} + +// HostnameDNSRecord is one DNS record a hostname depends on. +type HostnameDNSRecord struct { + // Name is the fully qualified name of the record, without a trailing dot. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=253 + Name string `json:"name"` + + // Type is the DNS record type. ALIAS stands for a CNAME at a zone apex, + // which DNS providers offer as ALIAS, ANAME or CNAME flattening. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=CNAME;ALIAS;TXT + Type string `json:"type"` + + // Content is the value the record must hold. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=512 + Content string `json:"content"` + + // Purpose says what the record is for. + // + // +kubebuilder:validation:Required + Purpose HostnameDNSRecordPurpose `json:"purpose"` + + // ManagedBy says who publishes the record: the user at their DNS provider, + // or the platform in a Datum DNS zone that serves the domain. + // + // +kubebuilder:validation:Required + ManagedBy HostnameDNSRecordManager `json:"managedBy"` + + // State says whether the record is in place. + // + // +kubebuilder:validation:Required + State HostnameDNSRecordState `json:"state"` } +// HostnameDNSRecordPurpose says what a DNS record is for. +// +// +kubebuilder:validation:Enum=Routing;Certificate;Ownership +type HostnameDNSRecordPurpose string + +const ( + // HostnameDNSRecordPurposeRouting points the hostname at the platform. + HostnameDNSRecordPurposeRouting HostnameDNSRecordPurpose = "Routing" + + // HostnameDNSRecordPurposeCertificate delegates the ACME DNS challenge for + // the hostname to the platform, so certificates issue before traffic moves. + HostnameDNSRecordPurposeCertificate HostnameDNSRecordPurpose = "Certificate" + + // HostnameDNSRecordPurposeOwnership proves ownership of the hostname's + // domain. It is listed until the domain is verified. + HostnameDNSRecordPurposeOwnership HostnameDNSRecordPurpose = "Ownership" +) + +// HostnameDNSRecordManager says who publishes a DNS record. +// +// +kubebuilder:validation:Enum=User;Platform +type HostnameDNSRecordManager string + +const ( + // HostnameDNSRecordManagedByUser means the user publishes the record at + // their DNS provider. + HostnameDNSRecordManagedByUser HostnameDNSRecordManager = "User" + + // HostnameDNSRecordManagedByPlatform means the platform publishes the record + // in a Datum DNS zone that serves the domain. + HostnameDNSRecordManagedByPlatform HostnameDNSRecordManager = "Platform" +) + +// HostnameDNSRecordState says whether a DNS record is in place. +// +// +kubebuilder:validation:Enum=Present;Missing +type HostnameDNSRecordState string + +const ( + // HostnameDNSRecordPresent means the record takes effect on the Internet. + HostnameDNSRecordPresent HostnameDNSRecordState = "Present" + + // HostnameDNSRecordMissing means the record is absent, wrong, or held in a + // zone that does not serve the domain. + HostnameDNSRecordMissing HostnameDNSRecordState = "Missing" +) + // HTTPProxyStatus defines the observed state of HTTPProxy. type HTTPProxyStatus struct { // Addresses lists the network addresses that have been bound to the diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index a4f1d80e..89b48e83 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -713,6 +713,21 @@ func (in *HTTPVerificationToken) DeepCopy() *HTTPVerificationToken { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostnameDNSRecord) DeepCopyInto(out *HostnameDNSRecord) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostnameDNSRecord. +func (in *HostnameDNSRecord) DeepCopy() *HostnameDNSRecord { + if in == nil { + return nil + } + out := new(HostnameDNSRecord) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostnameStatus) DeepCopyInto(out *HostnameStatus) { *out = *in @@ -723,6 +738,11 @@ func (in *HostnameStatus) DeepCopyInto(out *HostnameStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DNSRecords != nil { + in, out := &in.DNSRecords, &out.DNSRecords + *out = make([]HostnameDNSRecord, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostnameStatus. diff --git a/config/crd/bases/networking.datumapis.com_httpproxies.yaml b/config/crd/bases/networking.datumapis.com_httpproxies.yaml index 092defb6..af09208f 100644 --- a/config/crd/bases/networking.datumapis.com_httpproxies.yaml +++ b/config/crd/bases/networking.datumapis.com_httpproxies.yaml @@ -3705,6 +3705,66 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + dnsRecords: + description: |- + DNSRecords lists every DNS record this hostname depends on, who publishes + it, and whether it is in place. A record is Present only once it takes + effect on the Internet. Records the user publishes stay listed while the + hostname needs them, so this list alone says what is left to do. + items: + description: HostnameDNSRecord is one DNS record a hostname + depends on. + properties: + content: + description: Content is the value the record must hold. + maxLength: 512 + type: string + managedBy: + description: |- + ManagedBy says who publishes the record: the user at their DNS provider, + or the platform in a Datum DNS zone that serves the domain. + enum: + - User + - Platform + type: string + name: + description: Name is the fully qualified name of the record, + without a trailing dot. + maxLength: 253 + type: string + purpose: + description: Purpose says what the record is for. + enum: + - Routing + - Certificate + - Ownership + type: string + state: + description: State says whether the record is in place. + enum: + - Present + - Missing + type: string + type: + description: |- + Type is the DNS record type. ALIAS stands for a CNAME at a zone apex, + which DNS providers offer as ALIAS, ANAME or CNAME flattening. + enum: + - CNAME + - ALIAS + - TXT + type: string + required: + - content + - managedBy + - name + - purpose + - state + - type + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic hostname: description: |- Hostname is the fully qualified domain name being tracked. diff --git a/docs/api/httpproxies.md b/docs/api/httpproxies.md index 34c14506..5a1212c9 100644 --- a/docs/api/httpproxies.md +++ b/docs/api/httpproxies.md @@ -4725,6 +4725,16 @@ Must be a valid RFC 1123 hostname without a trailing dot.
Standard condition types include Verified and DNSRecordProgrammed.
false + + dnsRecords + []object + + DNSRecords lists every DNS record this hostname depends on, who publishes +it, and whether it is in place. A record is Present only once it takes +effect on the Internet. Records the user publishes stay listed while the +hostname needs them, so this list alone says what is left to do.
+ + false @@ -4804,3 +4814,75 @@ with respect to the current state of the instance.
false + + +### HTTPProxy.status.hostnameStatuses[index].dnsRecords[index] +[↩ Parent](#httpproxystatushostnamestatusesindex) + + + +HostnameDNSRecord is one DNS record a hostname depends on. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
contentstring + Content is the value the record must hold.
+ 		true
managedByenum + ManagedBy says who publishes the record: the user at their DNS provider, +or the platform in a Datum DNS zone that serves the domain.
+
+ Enum: User, Platform
+ 		true
namestring + Name is the fully qualified name of the record, without a trailing dot.
+ 		true
purposeenum + Purpose says what the record is for.
+
+ Enum: Routing, Certificate, Ownership
+ 		true
stateenum + State says whether the record is in place.
+
+ Enum: Present, Missing
+ 		true
typeenum + Type is the DNS record type. ALIAS stands for a CNAME at a zone apex, +which DNS providers offer as ALIAS, ANAME or CNAME flattening.
+
+ Enum: CNAME, ALIAS, TXT
+ 		true
diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index f95b0d4e..a8169b87 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -58,6 +58,8 @@ type HTTPProxyReconciler struct { Config config.NetworkServicesOperator DownstreamCluster cluster.Cluster + + routing *routingObserver } type desiredHTTPProxyResources struct { @@ -407,7 +409,9 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req applyPartialProgramming(ctx, desiredResources.partialProgramming, programmedCondition) - r.reconcileHTTPProxyHostnameStatus(ctx, cl.GetClient(), gateway, httpProxyCopy, string(req.ClusterName)) + if recheck := r.reconcileHTTPProxyHostnameStatus(ctx, cl.GetClient(), gateway, httpProxyCopy, string(req.ClusterName)); recheck { + return ctrl.Result{RequeueAfter: routingRecheckInterval}, nil + } return ctrl.Result{}, nil } @@ -571,21 +575,21 @@ func (r *HTTPProxyReconciler) reconcileHTTPProxyHostnameStatus( gateway *gatewayv1.Gateway, httpProxyCopy *networkingv1alpha.HTTPProxy, clusterName string, -) { +) (recheckRouting bool) { logger := log.FromContext(ctx) gatewayAcceptedCondition := apimeta.FindStatusCondition(gateway.Status.Conditions, string(gatewayv1.GatewayConditionAccepted)) if gatewayAcceptedCondition == nil { // Should never happen due to defaulting, but just in case logger.Info("accepted condition not found on gateway") - return + return false } else if gatewayAcceptedCondition.ObservedGeneration != gateway.Generation { logger.Info( "observed generation on accepted condition does not match generation on gateway, delaying processing", "gateway_generation", gateway.Generation, "condition_generation", gatewayAcceptedCondition.ObservedGeneration, ) - return + return false } logger.Info("updating hostname status") @@ -682,16 +686,22 @@ func (r *HTTPProxyReconciler) reconcileHTTPProxyHostnameStatus( availabilityStatuses := buildAvailabilityStatuses(acceptedHostnames, inUseHostnames, httpProxyCopy.Generation) dnsStatuses := r.buildDNSStatuses(ctx, cl, gateway, httpProxyCopy.Generation) certificateStatuses := r.buildCertificateStatuses(ctx, cl, clusterName, gateway, httpProxyCopy) + dnsRecordStatuses, recheckRouting := r.buildDNSRecordStatuses(ctx, cl, gateway, httpProxyCopy) previousHostnameStatuses := httpProxyCopy.Status.HostnameStatuses - httpProxyCopy.Status.HostnameStatuses = mergeHostnameStatuses(availabilityStatuses, dnsStatuses, certificateStatuses) + httpProxyCopy.Status.HostnameStatuses = mergeHostnameStatuses(availabilityStatuses, dnsStatuses, certificateStatuses, dnsRecordStatuses) preserveHostnameConditionTransitions(httpProxyCopy.Status.HostnameStatuses, previousHostnameStatuses) r.setCertificatesReadyCondition(httpProxyCopy, certificateStatuses, gateway) + + return recheckRouting } // SetupWithManager sets up the controller with the Manager. func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { r.mgr = mgr + if r.routing == nil { + r.routing = newRoutingObserver() + } builder := mcbuilder.ControllerManagedBy(mgr). For(&networkingv1alpha.HTTPProxy{}). @@ -1744,6 +1754,7 @@ func mergeHostnameStatuses(statusSets ...[]networkingv1alpha.HostnameStatus) []n for _, c := range hs.Conditions { apimeta.SetStatusCondition(&existing.Conditions, c) } + existing.DNSRecords = append(existing.DNSRecords, hs.DNSRecords...) } else { copy := hs byHostname[hs.Hostname] = © diff --git a/internal/controller/httpproxy_dns_records.go b/internal/controller/httpproxy_dns_records.go new file mode 100644 index 00000000..f71c9af9 --- /dev/null +++ b/internal/controller/httpproxy_dns_records.go @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "net" + "strings" + "sync" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + dnsutil "go.datum.net/network-services-operator/internal/util/dns" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" +) + +const ( + acmeChallengeLabel = "_acme-challenge" + + routingProbeLabel = "datum-routing-probe" + + routingObservationTTL = time.Minute + + routingRecheckInterval = 5 * time.Minute + + routingLookupTimeout = 2 * time.Second + + routingObservationCacheLimit = 10000 +) + +// routingObserver reports whether a hostname the user points at the platform +// resolves there, from public DNS, and remembers the answer briefly so a burst +// of reconciles costs one set of lookups. +type routingObserver struct { + lookupCNAME func(ctx context.Context, host string) (string, error) + lookupIP func(ctx context.Context, host string) ([]net.IPAddr, error) + now func() time.Time + + mu sync.Mutex + cache map[string]routingObservation +} + +type routingObservation struct { + present bool + expires time.Time +} + +func newRoutingObserver() *routingObserver { + return &routingObserver{ + lookupCNAME: net.DefaultResolver.LookupCNAME, + lookupIP: net.DefaultResolver.LookupIPAddr, + now: time.Now, + cache: map[string]routingObservation{}, + } +} + +func (o *routingObserver) routesTo(ctx context.Context, hostname, canonical string) bool { + key := hostname + "|" + canonical + now := o.now() + + o.mu.Lock() + if seen, ok := o.cache[key]; ok && now.Before(seen.expires) { + o.mu.Unlock() + return seen.present + } + o.mu.Unlock() + + present := o.lookup(ctx, hostname, canonical) + + o.mu.Lock() + defer o.mu.Unlock() + if len(o.cache) >= routingObservationCacheLimit { + for k, v := range o.cache { + if !now.Before(v.expires) { + delete(o.cache, k) + } + } + } + o.cache[key] = routingObservation{present: present, expires: now.Add(routingObservationTTL)} + return present +} + +func (o *routingObserver) lookup(ctx context.Context, hostname, canonical string) bool { + ctx, cancel := context.WithTimeout(ctx, routingLookupTimeout) + defer cancel() + + probe := hostname + if base, ok := strings.CutPrefix(hostname, "*."); ok { + probe = routingProbeLabel + "." + base + } + + if target, err := o.lookupCNAME(ctx, probe+"."); err == nil && strings.EqualFold(strings.TrimSuffix(target, "."), canonical) { + return true + } + + probeAddrs, err := o.lookupIP(ctx, probe+".") + if err != nil || len(probeAddrs) == 0 { + return false + } + canonicalAddrs, err := o.lookupIP(ctx, canonical+".") + if err != nil { + return false + } + for _, a := range probeAddrs { + for _, b := range canonicalAddrs { + if a.IP.Equal(b.IP) { + return true + } + } + } + return false +} + +// buildDNSRecordStatuses lists, for each custom hostname, the DNS records it +// depends on and whether each takes effect. It reports whether any record the +// user publishes for routing is still missing, since nothing but time tells +// the controller when one appears. +func (r *HTTPProxyReconciler) buildDNSRecordStatuses( + ctx context.Context, + cl client.Client, + gateway *gatewayv1.Gateway, + httpProxy *networkingv1alpha.HTTPProxy, +) (statuses []networkingv1alpha.HostnameStatus, recheckRouting bool) { + if !r.Config.Gateway.CertificateService.Enabled { + return nil, false + } + + logger := log.FromContext(ctx) + canonical := httpProxy.Status.CanonicalHostname + + var domains networkingv1alpha.DomainList + if err := cl.List(ctx, &domains, client.InNamespace(httpProxy.Namespace)); err != nil { + logger.Error(err, "failed to list domains for hostname DNS records") + return nil, false + } + + httpsListeners := map[string]gatewayv1.SectionName{} + for _, l := range gateway.Spec.Listeners { + if l.Protocol == gatewayv1.HTTPSProtocolType && l.Hostname != nil { + httpsListeners[string(*l.Hostname)] = l.Name + } + } + + for _, h := range httpProxy.Spec.Hostnames { + hostname := string(h) + if r.underTargetDomain(hostname) { + continue + } + + var records []networkingv1alpha.HostnameDNSRecord + + if canonical != "" { + routing := r.routingRecord(ctx, cl, gateway, hostname, canonical, domains.Items) + if routing.ManagedBy == networkingv1alpha.HostnameDNSRecordManagedByUser && routing.State == networkingv1alpha.HostnameDNSRecordMissing { + recheckRouting = true + } + records = append(records, routing) + } + + if listener, ok := httpsListeners[hostname]; ok && isSingleLabelWildcard(hostname) { + records = append(records, r.certificateRecords(ctx, cl, gateway, listener, hostname)...) + } + + if ownership, ok := ownershipRecord(hostname, domains.Items); ok { + records = append(records, ownership) + } + + if len(records) > 0 { + statuses = append(statuses, networkingv1alpha.HostnameStatus{Hostname: hostname, DNSRecords: records}) + } + } + + return statuses, recheckRouting +} + +func (r *HTTPProxyReconciler) underTargetDomain(hostname string) bool { + target := r.Config.Gateway.TargetDomain + return target != "" && (hostname == target || strings.HasSuffix(hostname, "."+target)) +} + +// routingRecord describes the record that points the hostname at the +// platform. A record the platform writes into a Datum DNS zone counts only +// while that zone is the one the domain's registry delegates to; any other +// record is judged by what public DNS answers. +func (r *HTTPProxyReconciler) routingRecord( + ctx context.Context, + cl client.Client, + gateway *gatewayv1.Gateway, + hostname, canonical string, + domains []networkingv1alpha.Domain, +) networkingv1alpha.HostnameDNSRecord { + record := networkingv1alpha.HostnameDNSRecord{ + Name: hostname, + Type: string(dnsv1alpha1.RRTypeCNAME), + Content: canonical, + Purpose: networkingv1alpha.HostnameDNSRecordPurposeRouting, + ManagedBy: networkingv1alpha.HostnameDNSRecordManagedByUser, + State: networkingv1alpha.HostnameDNSRecordMissing, + } + + if r.Config.Gateway.EnableDNSIntegration { + if present, rrType, managed := r.platformRoutingRecord(ctx, cl, gateway, hostname, domains); managed { + record.ManagedBy = networkingv1alpha.HostnameDNSRecordManagedByPlatform + record.Type = rrType + if present { + record.State = networkingv1alpha.HostnameDNSRecordPresent + } + return record + } + } + + for _, d := range domains { + if d.Spec.DomainName == hostname && d.Status.Apex { + record.Type = string(dnsv1alpha1.RRTypeALIAS) + break + } + } + + if r.routing != nil && r.routing.routesTo(ctx, hostname, canonical) { + record.State = networkingv1alpha.HostnameDNSRecordPresent + } + return record +} + +func (r *HTTPProxyReconciler) platformRoutingRecord( + ctx context.Context, + cl client.Client, + gateway *gatewayv1.Gateway, + hostname string, + domains []networkingv1alpha.Domain, +) (present bool, rrType string, managed bool) { + var recordSet dnsv1alpha1.DNSRecordSet + if err := cl.Get(ctx, client.ObjectKey{Namespace: gateway.Namespace, Name: dnsRecordSetName(gateway.Name, hostname)}, &recordSet); err != nil { + return false, "", false + } + if recordSet.Labels[labelManagedBy] != labelManagedByValue || recordSet.Annotations[annotationDNSHostname] != hostname { + return false, "", false + } + + rrType = string(recordSet.Spec.RecordType) + if !apimeta.IsStatusConditionTrue(recordSet.Status.Conditions, conditionTypeProgrammed) { + return false, rrType, true + } + + var zone dnsv1alpha1.DNSZone + if err := cl.Get(ctx, client.ObjectKey{Namespace: gateway.Namespace, Name: recordSet.Spec.DNSZoneRef.Name}, &zone); err != nil { + return false, rrType, true + } + domain, found := findDomainByName(domains, zone.Spec.DomainName) + if !found { + return false, rrType, true + } + return dnsutil.HasDNSAuthority(&domain, &zone), rrType, true +} + +// certificateRecords lists the record that delegates the hostname's ACME DNS +// challenge, when its certificate issues over DNS. Only the challenge name for +// this hostname is taken from the certificate's status. +func (r *HTTPProxyReconciler) certificateRecords( + ctx context.Context, + cl client.Client, + gateway *gatewayv1.Gateway, + listener gatewayv1.SectionName, + hostname string, +) []networkingv1alpha.HostnameDNSRecord { + var cert certificatesv1alpha1.TLSCertificate + if err := cl.Get(ctx, client.ObjectKey{Namespace: gateway.Namespace, Name: tlsCertificateName(gateway.Name, listener)}, &cert); err != nil { + return nil + } + + name := acmeChallengeLabel + "." + strings.TrimPrefix(hostname, "*.") + content := "" + for _, required := range cert.Status.RequiredDNSRecords { + if required.Purpose == certificatesv1alpha1.DNSRecordPurposeCertificate && + strings.EqualFold(strings.TrimSuffix(required.Name, "."), name) && + strings.EqualFold(required.Type, string(dnsv1alpha1.RRTypeCNAME)) { + content = required.Content + break + } + } + if content == "" && cert.Status.Issuance == certificatesv1alpha1.ChallengeTypeDNS01 { + content = cert.Status.DelegationTarget + } + if content == "" { + return nil + } + + state := networkingv1alpha.HostnameDNSRecordMissing + if delegation := apimeta.FindStatusCondition(cert.Status.Conditions, certificatesv1alpha1.ConditionDNSDelegationReady); delegation != nil && + delegation.Status == metav1.ConditionTrue { + state = networkingv1alpha.HostnameDNSRecordPresent + } + + return []networkingv1alpha.HostnameDNSRecord{{ + Name: name, + Type: string(dnsv1alpha1.RRTypeCNAME), + Content: strings.TrimSuffix(content, "."), + Purpose: networkingv1alpha.HostnameDNSRecordPurposeCertificate, + ManagedBy: networkingv1alpha.HostnameDNSRecordManagedByUser, + State: state, + }} +} + +// ownershipRecord returns the TXT record that would verify the most specific +// Domain covering the hostname, while no covering Domain is verified yet. +func ownershipRecord(hostname string, domains []networkingv1alpha.Domain) (networkingv1alpha.HostnameDNSRecord, bool) { + var pending *networkingv1alpha.Domain + for i := range domains { + d := &domains[i] + if !domainCoversHostname(d.Spec.DomainName, hostname) { + continue + } + if apimeta.IsStatusConditionTrue(d.Status.Conditions, networkingv1alpha.DomainConditionVerified) { + return networkingv1alpha.HostnameDNSRecord{}, false + } + if d.Status.Verification == nil || d.Status.Verification.DNSRecord.Name == "" { + continue + } + if pending == nil || len(d.Spec.DomainName) > len(pending.Spec.DomainName) { + pending = d + } + } + if pending == nil { + return networkingv1alpha.HostnameDNSRecord{}, false + } + + record := pending.Status.Verification.DNSRecord + return networkingv1alpha.HostnameDNSRecord{ + Name: strings.TrimSuffix(record.Name, "."), + Type: record.Type, + Content: record.Content, + Purpose: networkingv1alpha.HostnameDNSRecordPurposeOwnership, + ManagedBy: networkingv1alpha.HostnameDNSRecordManagedByUser, + State: networkingv1alpha.HostnameDNSRecordMissing, + }, true +} + +func domainCoversHostname(domainName, hostname string) bool { + if domainName == "" { + return false + } + return hostname == domainName || strings.HasSuffix(hostname, "."+domainName) +} diff --git a/internal/controller/httpproxy_dns_records_test.go b/internal/controller/httpproxy_dns_records_test.go new file mode 100644 index 00000000..17ed91da --- /dev/null +++ b/internal/controller/httpproxy_dns_records_test.go @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + certificatesv1alpha1 "go.datum.net/network-services-operator/internal/certificates/v1alpha1" + "go.datum.net/network-services-operator/internal/config" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" +) + +const testCanonicalHostname = "ruth-fourth-hrkgk.datumproxy.net" + +type fakeDNS struct { + cnames map[string]string + addrs map[string][]string + calls int +} + +func (f *fakeDNS) observer(now time.Time) *routingObserver { + return &routingObserver{ + lookupCNAME: func(_ context.Context, host string) (string, error) { + f.calls++ + if target, ok := f.cnames[host]; ok { + return target, nil + } + return "", errors.New("no such host") + }, + lookupIP: func(_ context.Context, host string) ([]net.IPAddr, error) { + f.calls++ + out := make([]net.IPAddr, 0, len(f.addrs[host])) + for _, a := range f.addrs[host] { + out = append(out, net.IPAddr{IP: net.ParseIP(a)}) + } + if len(out) == 0 { + return nil, errors.New("no such host") + } + return out, nil + }, + now: func() time.Time { return now }, + cache: map[string]routingObservation{}, + } +} + +func dnsRecordsTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, scheme.AddToScheme(s)) + require.NoError(t, gatewayv1.Install(s)) + require.NoError(t, networkingv1alpha.AddToScheme(s)) + require.NoError(t, certificatesv1alpha1.AddToScheme(s)) + require.NoError(t, dnsv1alpha1.AddToScheme(s)) + return s +} + +func dnsRecordsGateway(hostnames ...string) *gatewayv1.Gateway { + gw := &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: "s3"}} + for i, h := range hostnames { + gw.Spec.Listeners = append(gw.Spec.Listeners, + gatewayv1.Listener{Name: gatewayv1.SectionName("http-hostname-" + string(rune('0'+i))), Protocol: gatewayv1.HTTPProtocolType, Hostname: ptr.To(gatewayv1.Hostname(h))}, + gatewayv1.Listener{Name: gatewayv1.SectionName("https-hostname-" + string(rune('0'+i))), Protocol: gatewayv1.HTTPSProtocolType, Hostname: ptr.To(gatewayv1.Hostname(h))}, + ) + } + return gw +} + +func dnsRecordsProxy(hostnames ...string) *networkingv1alpha.HTTPProxy { + p := &networkingv1alpha.HTTPProxy{ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: "s3"}} + for _, h := range hostnames { + p.Spec.Hostnames = append(p.Spec.Hostnames, gatewayv1.Hostname(h)) + } + p.Status.CanonicalHostname = testCanonicalHostname + return p +} + +func verifiedDomain(name string, mutate ...func(*networkingv1alpha.Domain)) *networkingv1alpha.Domain { + d := &networkingv1alpha.Domain{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: name}, + Spec: networkingv1alpha.DomainSpec{DomainName: name}, + Status: networkingv1alpha.DomainStatus{Conditions: []metav1.Condition{{ + Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionTrue, Reason: networkingv1alpha.DomainReasonVerified, + }}}, + } + for _, m := range mutate { + m(d) + } + return d +} + +func pendingDomain(name, token string) *networkingv1alpha.Domain { + return &networkingv1alpha.Domain{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: name}, + Spec: networkingv1alpha.DomainSpec{DomainName: name}, + Status: networkingv1alpha.DomainStatus{ + Conditions: []metav1.Condition{{Type: networkingv1alpha.DomainConditionVerified, Status: metav1.ConditionFalse, Reason: networkingv1alpha.DomainReasonPendingVerification}}, + Verification: &networkingv1alpha.DomainVerificationStatus{ + DNSRecord: networkingv1alpha.DNSVerificationRecord{Name: "datum-custom-hostname." + name, Type: "TXT", Content: token}, + }, + }, + } +} + +func dns01Certificate(delegationReady bool, records ...certificatesv1alpha1.RequiredDNSRecord) *certificatesv1alpha1.TLSCertificate { + status := metav1.ConditionFalse + if delegationReady { + status = metav1.ConditionTrue + } + return &certificatesv1alpha1.TLSCertificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: tlsCertificateName("s3", "https-hostname-0")}, + Status: certificatesv1alpha1.TLSCertificateStatus{ + Issuance: certificatesv1alpha1.ChallengeTypeDNS01, + DelegationTarget: "k3f9q2x7.acme-dns.example.net", + RequiredDNSRecords: records, + Conditions: []metav1.Condition{{Type: certificatesv1alpha1.ConditionDNSDelegationReady, Status: status, Reason: "Checked"}}, + }, + } +} + +func recordsByPurpose(statuses []networkingv1alpha.HostnameStatus, hostname string) map[networkingv1alpha.HostnameDNSRecordPurpose]networkingv1alpha.HostnameDNSRecord { + out := map[networkingv1alpha.HostnameDNSRecordPurpose]networkingv1alpha.HostnameDNSRecord{} + for _, hs := range statuses { + if hs.Hostname != hostname { + continue + } + for _, r := range hs.DNSRecords { + out[r.Purpose] = r + } + } + return out +} + +func TestBuildDNSRecordStatuses(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC) + + datumZone := &dnsv1alpha1.DNSZone{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns", Name: "example-com"}, + Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "example.com"}, + Status: dnsv1alpha1.DNSZoneStatus{ + Nameservers: []string{"ns1.datumdomains.net"}, + Conditions: []metav1.Condition{ + {Type: "Accepted", Status: metav1.ConditionTrue, Reason: "Accepted"}, + {Type: "Programmed", Status: metav1.ConditionTrue, Reason: "Programmed"}, + }, + }, + } + platformRecordSet := func(hostname string, programmed bool) *dnsv1alpha1.DNSRecordSet { + status := metav1.ConditionFalse + if programmed { + status = metav1.ConditionTrue + } + return &dnsv1alpha1.DNSRecordSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: dnsRecordSetName("s3", hostname), + Labels: map[string]string{labelManagedBy: labelManagedByValue}, + Annotations: map[string]string{annotationDNSHostname: hostname}, + }, + Spec: dnsv1alpha1.DNSRecordSetSpec{ + DNSZoneRef: corev1.LocalObjectReference{Name: "example-com"}, + RecordType: dnsv1alpha1.RRTypeCNAME, + }, + Status: dnsv1alpha1.DNSRecordSetStatus{Conditions: []metav1.Condition{{Type: conditionTypeProgrammed, Status: status, Reason: "Test"}}}, + } + } + delegatedToDatum := func(d *networkingv1alpha.Domain) { + d.Status.Nameservers = []networkingv1alpha.Nameserver{{Hostname: "ns1.datumdomains.net"}} + } + delegatedElsewhere := func(d *networkingv1alpha.Domain) { + d.Status.Nameservers = []networkingv1alpha.Nameserver{{Hostname: "ns1.otherdns.example"}} + } + + tests := []struct { + name string + disabled bool + dnsIntegrated bool + hostnames []string + objects []client.Object + dns fakeDNS + wantRecheck bool + assert func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) + }{ + { + name: "nothing is published with the certificate service off", + disabled: true, + hostnames: []string{"*.s3.example.com"}, + objects: []client.Object{verifiedDomain("example.com")}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.Empty(t, statuses) + }, + }, + { + name: "a routing record the user has not published is missing and rechecked", + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.com")}, + wantRecheck: true, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + got := recordsByPurpose(statuses, "www.example.com") + assert.Equal(t, networkingv1alpha.HostnameDNSRecord{ + Name: "www.example.com", Type: "CNAME", Content: testCanonicalHostname, + Purpose: networkingv1alpha.HostnameDNSRecordPurposeRouting, ManagedBy: networkingv1alpha.HostnameDNSRecordManagedByUser, State: networkingv1alpha.HostnameDNSRecordMissing, + }, got[networkingv1alpha.HostnameDNSRecordPurposeRouting]) + assert.NotContains(t, got, networkingv1alpha.HostnameDNSRecordPurposeCertificate) + assert.NotContains(t, got, networkingv1alpha.HostnameDNSRecordPurposeOwnership) + }, + }, + { + name: "an exact hostname never lists a certificate record, since cert-manager issues it", + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.com"), dns01Certificate(false)}, + wantRecheck: true, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.NotContains(t, recordsByPurpose(statuses, "www.example.com"), networkingv1alpha.HostnameDNSRecordPurposeCertificate) + }, + }, + { + name: "a wildcard routes when a name beneath it resolves to the canonical hostname", + hostnames: []string{"*.s3.example.com"}, + objects: []client.Object{ + verifiedDomain("example.com"), + dns01Certificate(true), + }, + dns: fakeDNS{cnames: map[string]string{"datum-routing-probe.s3.example.com.": testCanonicalHostname + "."}}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + got := recordsByPurpose(statuses, "*.s3.example.com") + assert.Equal(t, "*.s3.example.com", got[networkingv1alpha.HostnameDNSRecordPurposeRouting].Name) + assert.Equal(t, networkingv1alpha.HostnameDNSRecordPresent, got[networkingv1alpha.HostnameDNSRecordPurposeRouting].State) + assert.Equal(t, networkingv1alpha.HostnameDNSRecord{ + Name: "_acme-challenge.s3.example.com", Type: "CNAME", Content: "k3f9q2x7.acme-dns.example.net", + Purpose: networkingv1alpha.HostnameDNSRecordPurposeCertificate, ManagedBy: networkingv1alpha.HostnameDNSRecordManagedByUser, State: networkingv1alpha.HostnameDNSRecordPresent, + }, got[networkingv1alpha.HostnameDNSRecordPurposeCertificate]) + }, + }, + { + name: "a flattened apex record that resolves to the platform's addresses is present", + hostnames: []string{"example.com"}, + objects: []client.Object{verifiedDomain("example.com", func(d *networkingv1alpha.Domain) { d.Status.Apex = true })}, + dns: fakeDNS{addrs: map[string][]string{ + "example.com.": {"203.0.113.10"}, + testCanonicalHostname + ".": {"203.0.113.10", "2001:db8::10"}, + }}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + got := recordsByPurpose(statuses, "example.com")[networkingv1alpha.HostnameDNSRecordPurposeRouting] + assert.Equal(t, "ALIAS", got.Type) + assert.Equal(t, networkingv1alpha.HostnameDNSRecordPresent, got.State) + }, + }, + { + name: "an address that is not the platform's leaves the routing record missing", + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.com")}, + dns: fakeDNS{addrs: map[string][]string{"www.example.com.": {"198.51.100.7"}, testCanonicalHostname + ".": {"203.0.113.10"}}}, + wantRecheck: true, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.Equal(t, networkingv1alpha.HostnameDNSRecordMissing, recordsByPurpose(statuses, "www.example.com")[networkingv1alpha.HostnameDNSRecordPurposeRouting].State) + }, + }, + { + name: "the certificate record is missing until the delegation resolves", + hostnames: []string{"*.s3.example.com"}, + objects: []client.Object{ + verifiedDomain("example.com"), + dns01Certificate(false, certificatesv1alpha1.RequiredDNSRecord{ + Name: "_acme-challenge.s3.example.com", Type: "CNAME", Content: "k3f9q2x7.acme-dns.example.net.", Purpose: certificatesv1alpha1.DNSRecordPurposeCertificate, + }), + }, + dns: fakeDNS{cnames: map[string]string{"datum-routing-probe.s3.example.com.": testCanonicalHostname + "."}}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + got := recordsByPurpose(statuses, "*.s3.example.com")[networkingv1alpha.HostnameDNSRecordPurposeCertificate] + assert.Equal(t, "k3f9q2x7.acme-dns.example.net", got.Content) + assert.Equal(t, networkingv1alpha.HostnameDNSRecordMissing, got.State) + }, + }, + { + name: "a required record for another name is not passed on", + hostnames: []string{"www.example.com"}, + objects: []client.Object{ + verifiedDomain("example.com"), + func() client.Object { + c := dns01Certificate(false, certificatesv1alpha1.RequiredDNSRecord{ + Name: "_acme-challenge.victim.example.org", Type: "CNAME", Content: "attacker.example.net", Purpose: certificatesv1alpha1.DNSRecordPurposeCertificate, + }) + c.Status.Issuance = certificatesv1alpha1.ChallengeTypeHTTP01 + c.Status.DelegationTarget = "" + return c + }(), + }, + wantRecheck: true, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.NotContains(t, recordsByPurpose(statuses, "www.example.com"), networkingv1alpha.HostnameDNSRecordPurposeCertificate) + }, + }, + { + name: "a platform record counts while Datum DNS serves the domain", + dnsIntegrated: true, + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.com", delegatedToDatum), datumZone, platformRecordSet("www.example.com", true)}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + got := recordsByPurpose(statuses, "www.example.com")[networkingv1alpha.HostnameDNSRecordPurposeRouting] + assert.Equal(t, networkingv1alpha.HostnameDNSRecordManagedByPlatform, got.ManagedBy) + assert.Equal(t, networkingv1alpha.HostnameDNSRecordPresent, got.State) + }, + }, + { + name: "a platform record in a zone the registry does not delegate to is missing", + dnsIntegrated: true, + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.com", delegatedElsewhere), datumZone, platformRecordSet("www.example.com", true)}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + got := recordsByPurpose(statuses, "www.example.com")[networkingv1alpha.HostnameDNSRecordPurposeRouting] + assert.Equal(t, networkingv1alpha.HostnameDNSRecordManagedByPlatform, got.ManagedBy) + assert.Equal(t, networkingv1alpha.HostnameDNSRecordMissing, got.State) + }, + }, + { + name: "a platform record not yet programmed is missing", + dnsIntegrated: true, + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.com", delegatedToDatum), datumZone, platformRecordSet("www.example.com", false)}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.Equal(t, networkingv1alpha.HostnameDNSRecordMissing, recordsByPurpose(statuses, "www.example.com")[networkingv1alpha.HostnameDNSRecordPurposeRouting].State) + }, + }, + { + name: "an unverified domain lists its ownership record on each hostname", + hostnames: []string{"www.example.com", "api.example.com"}, + objects: []client.Object{pendingDomain("example.com", "4f1c-token")}, + wantRecheck: true, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + want := networkingv1alpha.HostnameDNSRecord{ + Name: "datum-custom-hostname.example.com", Type: "TXT", Content: "4f1c-token", + Purpose: networkingv1alpha.HostnameDNSRecordPurposeOwnership, ManagedBy: networkingv1alpha.HostnameDNSRecordManagedByUser, State: networkingv1alpha.HostnameDNSRecordMissing, + } + assert.Equal(t, want, recordsByPurpose(statuses, "www.example.com")[networkingv1alpha.HostnameDNSRecordPurposeOwnership]) + assert.Equal(t, want, recordsByPurpose(statuses, "api.example.com")[networkingv1alpha.HostnameDNSRecordPurposeOwnership]) + }, + }, + { + name: "a verified domain that does not cover the hostname leaves its ownership record listed", + hostnames: []string{"www.example.com"}, + objects: []client.Object{verifiedDomain("example.org"), pendingDomain("example.com", "4f1c-token")}, + wantRecheck: true, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.Contains(t, recordsByPurpose(statuses, "www.example.com"), networkingv1alpha.HostnameDNSRecordPurposeOwnership) + }, + }, + { + name: "platform hostnames need no records", + hostnames: []string{"foo.datumproxy.net"}, + assert: func(t *testing.T, statuses []networkingv1alpha.HostnameStatus) { + assert.Empty(t, statuses) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cl := fake.NewClientBuilder().WithScheme(dnsRecordsTestScheme(t)).WithObjects(tt.objects...).Build() + r := &HTTPProxyReconciler{ + Config: config.NetworkServicesOperator{Gateway: config.GatewayConfig{ + TargetDomain: "datumproxy.net", + EnableDNSIntegration: tt.dnsIntegrated, + CertificateService: config.CertificateServiceConfig{Enabled: !tt.disabled}, + }}, + routing: tt.dns.observer(now), + } + + statuses, recheck := r.buildDNSRecordStatuses(context.Background(), cl, dnsRecordsGateway(tt.hostnames...), dnsRecordsProxy(tt.hostnames...)) + assert.Equal(t, tt.wantRecheck, recheck) + tt.assert(t, statuses) + }) + } +} + +func TestRoutingObserverRemembersAnswers(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC) + dns := &fakeDNS{cnames: map[string]string{"www.example.com.": testCanonicalHostname + "."}} + o := dns.observer(now) + + assert.True(t, o.routesTo(context.Background(), "www.example.com", testCanonicalHostname)) + calls := dns.calls + assert.True(t, o.routesTo(context.Background(), "www.example.com", testCanonicalHostname)) + assert.Equal(t, calls, dns.calls, "a second look inside the TTL must not hit DNS") + + o.now = func() time.Time { return now.Add(routingObservationTTL) } + delete(dns.cnames, "www.example.com.") + assert.False(t, o.routesTo(context.Background(), "www.example.com", testCanonicalHostname)) +} + +func TestMergeHostnameStatusesKeepsDNSRecords(t *testing.T) { + t.Parallel() + + routing := networkingv1alpha.HostnameDNSRecord{Name: "a.example.com", Type: "CNAME", Content: testCanonicalHostname, Purpose: networkingv1alpha.HostnameDNSRecordPurposeRouting} + merged := mergeHostnameStatuses( + []networkingv1alpha.HostnameStatus{{Hostname: "a.example.com", Conditions: []metav1.Condition{{Type: networkingv1alpha.HostnameConditionAvailable, Status: metav1.ConditionTrue}}}}, + []networkingv1alpha.HostnameStatus{{Hostname: "a.example.com", DNSRecords: []networkingv1alpha.HostnameDNSRecord{routing}}}, + ) + + require.Len(t, merged, 1) + assert.Len(t, merged[0].Conditions, 1) + assert.Equal(t, []networkingv1alpha.HostnameDNSRecord{routing}, merged[0].DNSRecords) +}