From 38131ed283915a6df7c2b711ccc0efd835cdc06b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 24 Sep 2026 18:10:49 -0500 Subject: [PATCH] fix: Collect a projected interface before its name is reused An interface is published under a name a replacement reuses on purpose, and the network a published copy names cannot be changed after it is created. So a copy that outlived the interface it was published from held the name against every replacement that followed, and the replacement was never published at all. A consumer kept reading the interface their previous instance had, and a network service resolved its members to it. Nothing collected the copy because nothing was holding the interface open long enough to say where its copy had gone. A published interface that simply vanished took with it the labels naming the project its copy was handed to. Ordered teardown is the whole of the fix. Nothing here detects a copy that outlived its interface, or takes a name back from one, because a copy that reaches that state is a teardown that did not run rather than a case to recover from. Key changes: - Hold a published interface on the hub with a finalizer until the copy handed to its project has been collected, so the project's copy never outlives the interface it describes - Wait for a teardown still in flight rather than writing through the copy it is collecting - Refuse to write over an interface in a project namespace that is not a published copy Fixes #495 Co-Authored-By: Claude Opus 5 (1M context) --- .../controller/networkinterface_projection.go | 108 ++++++-- .../networkinterface_projection_gc.go | 15 +- .../networkinterface_projection_test.go | 199 ++++++++++++++- .../controller/networkinterface_projector.go | 83 +++++- .../controller/networkinterface_writeback.go | 14 +- .../chainsaw-test.yaml | 237 ++++++++++++++++++ 6 files changed, 620 insertions(+), 36 deletions(-) create mode 100644 test/e2e/networkinterface-name-reuse-across-networks/chainsaw-test.yaml diff --git a/internal/controller/networkinterface_projection.go b/internal/controller/networkinterface_projection.go index c6a7cee7..a3249f48 100644 --- a/internal/controller/networkinterface_projection.go +++ b/internal/controller/networkinterface_projection.go @@ -6,8 +6,10 @@ import ( "context" "fmt" "maps" + "time" apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -73,13 +75,81 @@ func isProjection(iface *networkingv1alpha.NetworkInterface) bool { return iface.Labels[networkingv1alpha.NetworkInterfaceProjectionLabel] == "true" } -// writeProjection converges a copy onto what the source says. The copy is -// overwritten in full on every pass, so an edit to it never survives and never -// reaches the source: no controller reads a copy to decide anything. +// projectionSlotRetry paces a writer that found the name it publishes under +// still held. A teardown takes one reconcile of the controller holding it, so +// what is waited on here is measured in seconds. +const projectionSlotRetry = 5 * time.Second + +// projectionSlotHeld reports that the copy occupying a name is still being torn +// down, so the name is not free to write yet. +// +// A copy is collected while the interface it was published from is still held, +// and the name is free once that has finished. A writer that arrives mid +// teardown waits for it rather than writing through what is going. +type projectionSlotHeld struct { + key client.ObjectKey +} + +func (e *projectionSlotHeld) Error() string { + return fmt.Sprintf("the interface copy at %s has not finished being torn down", e.key) +} + +// inspectProjectionSlot reports the copy occupying a name. +func inspectProjectionSlot( + ctx context.Context, + cl client.Client, + key client.ObjectKey, +) (*networkingv1alpha.NetworkInterface, error) { + var occupant networkingv1alpha.NetworkInterface + if err := cl.Get(ctx, key, &occupant); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("failed reading the interface copy: %w", err) + } + + // Nothing here owns an interface somebody else made, and an interface that is + // not a copy is the only thing in this namespace that could be holding the + // name for a reason. + if !isProjection(&occupant) { + return nil, fmt.Errorf( + "an interface that is not a published copy already holds %s, refusing to write over it", key) + } + + return &occupant, nil +} + +// applyProjection converges a copy onto what the source says. // // A label the source has dropped is removed from the copy rather than left in // place. A copy is selected by its labels, so one the source no longer carries // is a member of a service the interface has left. +func applyProjection( + copied *networkingv1alpha.NetworkInterface, + desired *networkingv1alpha.NetworkInterface, + extraLabels map[string]string, + owner func(*networkingv1alpha.NetworkInterface) error, +) error { + wanted := map[string]string{} + maps.Copy(wanted, desired.Labels) + maps.Copy(wanted, extraLabels) + + copied.Labels, _ = replaceOwnedLabels(copied.Labels, wanted, platformLabelPrefixes, nil) + copied.Spec = *desired.Spec.DeepCopy() + if owner != nil { + return owner(copied) + } + return nil +} + +// writeProjection converges a copy onto what the source says. The copy is +// overwritten in full on every pass, so an edit to it never survives and never +// reaches the source: no controller reads a copy to decide anything. +// +// The name an interface is published under is one a replacement reuses. Nothing +// here empties the name first, because the copy that held it is collected while +// the interface it was published from is still there to say where it went, and a +// replacement that arrives before that has finished waits for it. func writeProjection( ctx context.Context, cl client.Client, @@ -88,21 +158,23 @@ func writeProjection( extraLabels map[string]string, owner func(*networkingv1alpha.NetworkInterface) error, ) error { - copied := &networkingv1alpha.NetworkInterface{ - ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: namespace}, + key := client.ObjectKey{Namespace: namespace, Name: desired.Name} + + occupant, err := inspectProjectionSlot(ctx, cl, key) + if err != nil { + return err + } + // A copy on its way out is not something to write through. What is left of it + // goes when its teardown finishes, and the name is free then. + if occupant != nil && !occupant.DeletionTimestamp.IsZero() { + return &projectionSlotHeld{key: key} } + copied := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + } if _, err := controllerutil.CreateOrUpdate(ctx, cl, copied, func() error { - wanted := map[string]string{} - maps.Copy(wanted, desired.Labels) - maps.Copy(wanted, extraLabels) - - copied.Labels, _ = replaceOwnedLabels(copied.Labels, wanted, platformLabelPrefixes, nil) - copied.Spec = *desired.Spec.DeepCopy() - if owner != nil { - return owner(copied) - } - return nil + return applyProjection(copied, desired, extraLabels, owner) }); err != nil { return fmt.Errorf("failed writing the interface copy: %w", err) } @@ -119,9 +191,9 @@ func writeProjection( return nil } -// collectProjection removes a copy, and only ever a copy. A copy holds no -// finalizer and nothing real depends on it, so removing one cannot strand an -// address or hold a namespace open. +// collectProjection removes a copy, and only ever a copy. Nothing real depends +// on a copy and a copy in a project control plane holds no finalizer, so +// removing one cannot strand an address or hold a namespace open. func collectProjection(ctx context.Context, cl client.Client, key client.ObjectKey) error { var copied networkingv1alpha.NetworkInterface if err := cl.Get(ctx, key, &copied); err != nil { diff --git a/internal/controller/networkinterface_projection_gc.go b/internal/controller/networkinterface_projection_gc.go index a58b0791..4cd388c7 100644 --- a/internal/controller/networkinterface_projection_gc.go +++ b/internal/controller/networkinterface_projection_gc.go @@ -20,11 +20,16 @@ import ( // NetworkInterfaceProjectionGCReconciler removes an interface copy from a // project control plane once nothing is published behind it. // -// It is the projector's other half, and it exists because the projector is -// keyed on the published copy: an object that has gone carries no labels, so -// nothing on the hub can say which project a vanished copy belonged to. Keying -// on the copy instead answers that from the copy itself, and it also brings a -// copy back that somebody deleted by hand. +// It is the projector's backstop rather than its cleanup path: the projector +// holds a published interface until its copy is collected, so the ordinary +// deletion never reaches here. What does reach here is a copy that outlived a +// deletion nothing was there to hold, and a copy somebody deleted by hand, both +// of which are answered from the copy itself. +// +// It asks only whether anything is published behind a copy. It deliberately does +// not compare what the copy was published from: it reads the hub through a cache +// that lags, and a copy that had just been replaced would look superseded to a +// lagging read, so identity is the writer's question and presence is this one's. type NetworkInterfaceProjectionGCReconciler struct { Projects ProjectClusterResolver diff --git a/internal/controller/networkinterface_projection_test.go b/internal/controller/networkinterface_projection_test.go index 3a67402b..cc3b854e 100644 --- a/internal/controller/networkinterface_projection_test.go +++ b/internal/controller/networkinterface_projection_test.go @@ -21,6 +21,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -191,6 +192,21 @@ func newVisibility(t *testing.T) *visibility { // interfaceOnCell writes the interface a cell holds, addresses and all, in the // shape the claim controller leaves behind. func (v *visibility) interfaceOnCell() *networkingv1alpha.NetworkInterface { + v.t.Helper() + return v.interfaceOnCellForNetwork("default") +} + +// networkInProject writes another network a consumer can move an instance onto. +func (v *visibility) networkInProject(name string) { + v.t.Helper() + network := &networkingv1alpha.Network{} + network.Name = name + network.Namespace = v.projectNamespace + network.Spec.IPAM.Mode = networkingv1alpha.NetworkIPAMModeAuto + require.NoError(v.t, v.project.Create(v.ctx, network)) +} + +func (v *visibility) interfaceOnCellForNetwork(network string) *networkingv1alpha.NetworkInterface { v.t.Helper() name := boundInterfaceName @@ -198,7 +214,7 @@ func (v *visibility) interfaceOnCell() *networkingv1alpha.NetworkInterface { iface.Name = name iface.Namespace = v.cellNamespace iface.Spec = networkingv1alpha.NetworkInterfaceSpec{ - Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Network: networkingv1alpha.LocalNetworkRef{Name: network}, ClaimRef: &networkingv1alpha.NetworkInterfaceClaimRef{Name: name}, InterfaceName: "eth0", MTU: 1440, @@ -236,20 +252,57 @@ func (v *visibility) interfaceOnCell() *networkingv1alpha.NetworkInterface { return iface } +// holdOnCell puts on the interface the finalizer the claim controller holds it +// with, so deleting it leaves it on its way out rather than gone at once. +func (v *visibility) holdOnCell(iface *networkingv1alpha.NetworkInterface) { + v.t.Helper() + iface.Finalizers = append(iface.Finalizers, networkInterfaceFinalizer) + require.NoError(v.t, v.cell.Update(v.ctx, iface)) +} + +func (v *visibility) releaseOnCell(iface *networkingv1alpha.NetworkInterface) { + v.t.Helper() + var held networkingv1alpha.NetworkInterface + require.NoError(v.t, v.cell.Get(v.ctx, client.ObjectKeyFromObject(iface), &held)) + held.Finalizers = nil + require.NoError(v.t, v.cell.Update(v.ctx, &held)) +} + func (v *visibility) publish() { v.t.Helper() - name := boundInterfaceName - require.NoError(v.t, v.writeBack.publish(v.ctx, v.cell, client.ObjectKey{ - Namespace: v.cellNamespace, Name: name, - })) + require.NoError(v.t, v.publishErr()) +} + +func (v *visibility) publishErr() error { + v.t.Helper() + return v.writeBack.publish(v.ctx, v.cell, client.ObjectKey{ + Namespace: v.cellNamespace, Name: boundInterfaceName, + }) +} + +// reconcileHub runs the projector's own entry point rather than its inner +// projection, which is the only way the hold it takes on a published interface +// is exercised. +func (v *visibility) reconcileHub() error { + v.t.Helper() + _, err := v.projector.Reconcile(v.ctx, ctrl.Request{NamespacedName: client.ObjectKey{ + Namespace: v.cellNamespace, Name: boundInterfaceName, + }}) + return err } func (v *visibility) handToProject() { v.t.Helper() - name := boundInterfaceName + require.NoError(v.t, v.handToProjectErr()) +} + +func (v *visibility) handToProjectErr() error { + v.t.Helper() var published networkingv1alpha.NetworkInterface - require.NoError(v.t, v.hub.Get(v.ctx, client.ObjectKey{Namespace: v.cellNamespace, Name: name}, &published)) - require.NoError(v.t, v.projector.project(v.ctx, &published)) + require.NoError(v.t, v.hub.Get(v.ctx, client.ObjectKey{ + Namespace: v.cellNamespace, Name: boundInterfaceName, + }, &published)) + return v.projector.project(v.ctx, &published) } func (v *visibility) collect(name string) { @@ -623,3 +676,133 @@ func TestACopyLosesALabelItsSourceDropped(t *testing.T) { require.NotContains(t, copied.Labels, "compute.datumapis.com/workload-name") require.Equal(t, testLocationName, copied.Labels[networkingv1alpha.NetworkInterfaceLocationLabel]) } + +// A replacement reuses the name its predecessor was published under on purpose, +// and it can land on another network. The copy a consumer reads is collected +// before the name is written again: the network a copy names cannot be moved, so +// a copy that outlived its interface would hold the name against every +// replacement that followed. +func TestARecreatedInterfaceOnAnotherNetworkTakesTheName(t *testing.T) { + v := newVisibility(t) + v.networkInProject("net-b") + + first := v.interfaceOnCell() + v.holdOnCell(first) + v.publish() + require.NoError(t, v.reconcileHub()) + + copied, found := v.projectCopy() + require.True(t, found) + require.Equal(t, "default", copied.Spec.Network.Name) + + // The interface goes the way the claim controller releases one: held by its own + // finalizer, so the write-back sees it on its way out. + require.NoError(t, v.cell.Delete(v.ctx, first)) + + v.publish() + published, found := v.hubCopy(boundInterfaceName) + require.True(t, found, "the published copy is held for the copy it handed to the project") + require.False(t, published.DeletionTimestamp.IsZero()) + + require.NoError(t, v.reconcileHub()) + _, found = v.projectCopy() + require.False(t, found, "the copy a consumer reads goes before the name is reused") + _, found = v.hubCopy(boundInterfaceName) + require.False(t, found) + + v.releaseOnCell(first) + + second := v.interfaceOnCellForNetwork("net-b") + require.NotEqual(t, first.UID, second.UID) + + // Nothing is left holding the name, so the replacement converges without + // anything having to be taken from it. + v.publish() + require.NoError(t, v.reconcileHub()) + + published, found = v.hubCopy(boundInterfaceName) + require.True(t, found) + require.Equal(t, "net-b", published.Spec.Network.Name, "the published copy follows the interface") + + copied, found = v.projectCopy() + require.True(t, found, "a consumer sees the interface behind the instance they have now") + require.Equal(t, "net-b", copied.Spec.Network.Name) + require.Len(t, copied.OwnerReferences, 1) + require.Equal(t, "net-b", copied.OwnerReferences[0].Name, "the copy follows the network it belongs to") +} + +// A replacement that stays on its network is the ordinary case, and the slot is +// converged rather than emptied and remade. +func TestARecreatedInterfaceOnTheSameNetworkKeepsItsCopy(t *testing.T) { + v := newVisibility(t) + + first := v.interfaceOnCell() + v.publish() + v.handToProject() + + before, found := v.projectCopy() + require.True(t, found) + + require.NoError(t, v.cell.Delete(v.ctx, first)) + v.interfaceOnCell() + + v.publish() + v.handToProject() + + after, found := v.projectCopy() + require.True(t, found) + require.Equal(t, before.UID, after.UID, "a copy nothing needs to remake is not remade") +} + +// An interface that is not a copy is the only thing that could be holding a name +// for a reason. Taking the name from it would delete something a consumer made. +func TestAnInterfaceThatIsNotACopyKeepsItsName(t *testing.T) { + v := newVisibility(t) + v.interfaceOnCell() + v.publish() + + theirs := &networkingv1alpha.NetworkInterface{} + theirs.Name = boundInterfaceName + theirs.Namespace = v.projectNamespace + theirs.Spec.Network = networkingv1alpha.LocalNetworkRef{Name: "default"} + theirs.Spec.InterfaceName = "eth7" + require.NoError(t, v.project.Create(v.ctx, theirs)) + + require.Error(t, v.handToProjectErr(), "the slot is not the projector's to take") + + kept, found := v.projectCopy() + require.True(t, found) + require.Equal(t, "eth7", kept.Spec.InterfaceName) + require.Equal(t, theirs.UID, kept.UID) +} + +// Nothing replays a deletion, so the copy a project holds has to be collected +// while the interface it was published from is still there to say where it went. +func TestAPublishedInterfaceIsHeldUntilItsCopyIsCollected(t *testing.T) { + v := newVisibility(t) + v.interfaceOnCell() + v.publish() + + require.NoError(t, v.reconcileHub()) + + published, found := v.hubCopy(boundInterfaceName) + require.True(t, found) + require.Contains(t, published.Finalizers, networkInterfaceProjectionFinalizer) + _, found = v.projectCopy() + require.True(t, found) + + require.NoError(t, v.hub.Delete(v.ctx, published)) + + published, found = v.hubCopy(boundInterfaceName) + require.True(t, found, "the published copy is held until its copy is gone") + require.False(t, published.DeletionTimestamp.IsZero()) + _, found = v.projectCopy() + require.True(t, found) + + require.NoError(t, v.reconcileHub()) + + _, found = v.projectCopy() + require.False(t, found, "the copy a consumer reads goes with the interface") + _, found = v.hubCopy(boundInterfaceName) + require.False(t, found, "the hold is released once nothing is left behind") +} diff --git a/internal/controller/networkinterface_projector.go b/internal/controller/networkinterface_projector.go index 13e8d3e3..7e51f1dd 100644 --- a/internal/controller/networkinterface_projector.go +++ b/internal/controller/networkinterface_projector.go @@ -32,6 +32,16 @@ type NetworkInterfaceProjector struct { hub client.Client } +// networkInterfaceProjectionFinalizer holds a published interface on the hub +// until the copy it was handed to a project has been collected. +// +// Nothing replays a deletion. A published interface that simply vanished would +// take with it the labels naming the project its copy went to, leaving a copy +// that describes an interface that no longer exists and no event anywhere able +// to say where it is. Holding the original until its copy is gone means the one +// controller that knows both planes does the cleanup while it still can. +const networkInterfaceProjectionFinalizer = "networking.datumapis.com/networkinterface-projection" + // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch @@ -39,16 +49,39 @@ type NetworkInterfaceProjector struct { func (r *NetworkInterfaceProjector) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var published networkingv1alpha.NetworkInterface if err := r.hub.Get(ctx, req.NamespacedName, &published); err != nil { - // A published interface that has gone leaves nothing here to route by. - // The project-plane collector is what removes what it left behind. + // A published interface that has finished going leaves nothing here to + // route by. Its copy was collected while it was still held, under the + // finalizer below. return ctrl.Result{}, client.IgnoreNotFound(err) } - if !isProjection(&published) || !published.DeletionTimestamp.IsZero() { + if !isProjection(&published) { return ctrl.Result{}, nil } - return ctrl.Result{}, r.project(ctx, &published) + if !published.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.teardown(ctx, &published) + } + + if controllerutil.AddFinalizer(&published, networkInterfaceProjectionFinalizer) { + if err := r.hub.Update(ctx, &published); err != nil { + return ctrl.Result{}, fmt.Errorf("failed holding the published interface for its copy: %w", err) + } + } + + // A name held by a copy that is being torn down is a state to come back to, + // not a reconcile that failed. + if err := r.project(ctx, &published); err != nil { + var held *projectionSlotHeld + if errors.As(err, &held) { + log.FromContext(ctx).Info("waiting for the copy holding this name to finish being torn down", + "copy", held.key.String()) + return ctrl.Result{RequeueAfter: projectionSlotRetry}, nil + } + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil } func (r *NetworkInterfaceProjector) project( @@ -104,6 +137,48 @@ func (r *NetworkInterfaceProjector) project( return writeProjection(ctx, projectClient, namespace, desired, nil, owner) } +// teardown collects the copy a published interface on its way out was handed to +// a project, then lets the original go. +func (r *NetworkInterfaceProjector) teardown( + ctx context.Context, + published *networkingv1alpha.NetworkInterface, +) error { + if !controllerutil.ContainsFinalizer(published, networkInterfaceProjectionFinalizer) { + return nil + } + + project := downstreamclient.UpstreamClusterNameFromLabel( + published.Labels[downstreamclient.UpstreamOwnerClusterNameLabel]) + namespace := published.Labels[downstreamclient.UpstreamOwnerNamespaceLabel] + + // An interface naming no project was never handed to one, and a project the + // manager no longer engages has no control plane left holding a copy. Neither + // leaves anything to collect, and neither may hold the original open. + if project != "" && namespace != "" { + projectClient, err := r.Projects.ClientForProject(ctx, project) + switch { + case err == nil: + key := client.ObjectKey{Namespace: namespace, Name: published.Name} + if err := collectProjection(ctx, projectClient, key); err != nil { + return err + } + case errors.Is(err, multicluster.ErrClusterNotFound): + log.FromContext(ctx).Info("releasing a published interface whose project is no longer engaged", + "project", project) + default: + return fmt.Errorf("failed reaching project %q: %w", project, err) + } + } + + if controllerutil.RemoveFinalizer(published, networkInterfaceProjectionFinalizer) { + if err := r.hub.Update(ctx, published); err != nil { + return fmt.Errorf("failed releasing the published interface: %w", err) + } + } + + return nil +} + // SetupWithManager registers the projector against the hub the manager runs in. func (r *NetworkInterfaceProjector) SetupWithManager(mgr manager.Manager) error { r.hub = mgr.GetClient() diff --git a/internal/controller/networkinterface_writeback.go b/internal/controller/networkinterface_writeback.go index b211ce20..9429ced5 100644 --- a/internal/controller/networkinterface_writeback.go +++ b/internal/controller/networkinterface_writeback.go @@ -58,7 +58,19 @@ func (r *NetworkInterfaceWriteBackReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, err } - return ctrl.Result{}, r.publish(ctx, cl.GetClient(), req.NamespacedName) + // A name held by a copy that is being torn down is a state to come back to, + // not a reconcile that failed. + if err := r.publish(ctx, cl.GetClient(), req.NamespacedName); err != nil { + var held *projectionSlotHeld + if errors.As(err, &held) { + log.FromContext(ctx).Info("waiting for the copy holding this name to finish being torn down", + "copy", held.key.String()) + return ctrl.Result{RequeueAfter: projectionSlotRetry}, nil + } + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil } func (r *NetworkInterfaceWriteBackReconciler) publish( diff --git a/test/e2e/networkinterface-name-reuse-across-networks/chainsaw-test.yaml b/test/e2e/networkinterface-name-reuse-across-networks/chainsaw-test.yaml new file mode 100644 index 00000000..907a80de --- /dev/null +++ b/test/e2e/networkinterface-name-reuse-across-networks/chainsaw-test.yaml @@ -0,0 +1,237 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +# +# A workload deleted and recreated under the same name holds the same interface +# name: the name is a slot a replacement reuses on purpose. A replacement can +# also land on a different network, and the network a published copy names cannot +# be changed after it is created. So a copy that outlives the interface it was +# published from holds the name against every replacement that follows, and the +# replacement is never published at all. +# +# This asserts the copy is collected when its interface goes, that the name is +# free afterwards, and that a replacement on another network is published under +# it. The three together are what keep the name reusable. +# +# This covers the cell-to-hub hop. Handing a published copy on to a project +# control plane is the projector's half: in this env the cell publishes to the +# Karmada hub while the manager watches the cluster it is deployed on, so the +# projector sees nothing here and the project-plane copy has no coverage. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: networkinterface-name-reuse-across-networks +spec: + cluster: nso-standard + steps: + - name: Deliver both networks' presence to this location + description: | + Two networks, so the replacement can land on one its predecessor never + used. Each needs a presence at the location whose cell allocates the + address. + try: + - apply: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkContext + metadata: + name: nic-reuse-net-a-us-central-1 + namespace: ipam-e2e-alpha + spec: + network: + name: nic-reuse-net-a + location: + name: us-central-1 + ipFamilies: + - IPv6 + mtu: 1440 + - apply: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkContext + metadata: + name: nic-reuse-net-b-us-central-1 + namespace: ipam-e2e-alpha + spec: + network: + name: nic-reuse-net-b + location: + name: us-central-1 + ipFamilies: + - IPv6 + mtu: 1440 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-reuse-net-a + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv6 + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: Network + metadata: + name: nic-reuse-net-b + namespace: ipam-e2e-alpha + spec: + ipam: + mode: Auto + ipFamilies: + - IPv6 + + - name: The first interface is published under its name + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + spec: + network: + name: nic-reuse-net-a + ipFamilies: + - IPv6 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - assert: + cluster: nso-hub + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + labels: + networking.datumapis.com/network-interface-projection: "true" + networking.datumapis.com/location: us-central-1 + spec: + network: + name: nic-reuse-net-a + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/component=cell-controller-manager --tail=100 + + - name: Releasing the interface frees the name + description: | + The copy goes with the interface it was published from. Nothing replays a + deletion, so a copy left behind here is a copy nothing would ever collect. + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-reuse + namespace: ipam-e2e-alpha + - error: + cluster: nso-hub + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + + - name: A replacement on the other network takes the name + description: | + The same claim name, and so the same interface name, on a network the + first interface never used. This is the case that wedged the name: the + copy that was there could not be moved onto the new network, and until it + was collected the replacement had nowhere to be published. + try: + - create: + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + spec: + network: + name: nic-reuse-net-b + ipFamilies: + - IPv6 + - assert: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + status: + (conditions[?type == 'Allocated'] | [0].status): "True" + - assert: + cluster: nso-hub + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + labels: + networking.datumapis.com/network-interface-projection: "true" + spec: + network: + name: nic-reuse-net-b + catch: + - script: + content: | + kubectl -n ipam-e2e-alpha get networkinterfaceclaim,networkinterface -o yaml + kubectl -n network-services-operator-system logs -l app.kubernetes.io/component=cell-controller-manager --tail=100 + + - name: Releasing the replacement leaves nothing behind + try: + - delete: + timeout: 120s + ref: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterfaceClaim + name: nic-reuse + namespace: ipam-e2e-alpha + - error: + cluster: nso-hub + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + - error: + timeout: 120s + resource: + apiVersion: networking.datumapis.com/v1alpha + kind: NetworkInterface + metadata: + name: nic-reuse + namespace: ipam-e2e-alpha + cleanup: + - script: + content: | + set -u + IPAM=$(git rev-parse --show-toplevel)/hack/ipam-project-kubectl.sh + kubectl -n ipam-e2e-alpha delete networkinterfaceclaim nic-reuse --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete networkinterface nic-reuse --ignore-not-found --timeout=60s + "$IPAM" project-alpha -n default delete ipclaim nic-reuse-f-ipv6 --ignore-not-found --timeout=60s + kubectl -n ipam-e2e-alpha delete network nic-reuse-net-a nic-reuse-net-b --ignore-not-found --timeout=120s + kubectl -n ipam-e2e-alpha delete networkcontext nic-reuse-net-a-us-central-1 nic-reuse-net-b-us-central-1 --ignore-not-found --timeout=120s