From 5f1e58d668ab7a6acec18de03cc90430cde6d7b5 Mon Sep 17 00:00:00 2001 From: Qishuai Liu Date: Sun, 11 Oct 2026 23:17:37 +0900 Subject: [PATCH 1/2] Add -nat64 bpf, a translator of sixup's own that needs no module A tc BPF program on the egress of the dummy device sixup-nat64 translates statelessly (RFC 7915): each IPv6 source gets an address of its own from 198.18.0.0/15, and conntrack and the source NAT on the way out do the stateful part, as for any IPv4 host behind the router. Translated packets go back to the device's ingress. TCP, UDP, ICMP and ICMP errors with their quotes are translated, fragments one by one, behind a prefix of any length RFC 6052 allows. Fragmented ICMP and IPv4 UDP without a checksum need the whole datagram and go through a slow path in sixup, which reassembles, translates and sends them back marked. Mappings conntrack has no flow for are removed. Linux 6.6 or later is needed for tcx. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 2 +- bpf/nat64.c | 800 ++++++++++++++++++++++++++++++++++ firewall.go | 2 +- flags.go | 2 +- go.mod | 1 + go.sum | 5 + main.go | 21 +- nat64_bpfeb.go | 192 ++++++++ nat64_bpfeb.o | Bin 0 -> 70672 bytes nat64_bpfel.go | 192 ++++++++ nat64_bpfel.o | Bin 0 -> 71648 bytes nat64bpf.go | 346 +++++++++++++++ nat64bpf_integration_test.go | 487 +++++++++++++++++++++ nat64bpf_stub.go | 16 + nat64slow.go | 420 ++++++++++++++++++ nat64slow_integration_test.go | 658 ++++++++++++++++++++++++++++ nat64slow_test.go | 189 ++++++++ 17 files changed, 3324 insertions(+), 9 deletions(-) create mode 100644 bpf/nat64.c create mode 100644 nat64_bpfeb.go create mode 100644 nat64_bpfeb.o create mode 100644 nat64_bpfel.go create mode 100644 nat64_bpfel.o create mode 100644 nat64bpf.go create mode 100644 nat64bpf_integration_test.go create mode 100644 nat64bpf_stub.go create mode 100644 nat64slow.go create mode 100644 nat64slow_integration_test.go create mode 100644 nat64slow_test.go diff --git a/Dockerfile b/Dockerfile index 1aba5f1..e2f96e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG TARGETARCH TARGETVARIANT VERSION=dev WORKDIR /src COPY go.mod go.sum ./ RUN go mod download -COPY *.go LICENSE NOTICE ./ +COPY *.go *.o LICENSE NOTICE ./ RUN CGO_ENABLED=0 GOOS=linux GOARCH="$TARGETARCH" GOARM="${TARGETVARIANT#v}" \ go build -trimpath -ldflags="-s -w -X main.version=${VERSION}" -o /out/sixup . diff --git a/bpf/nat64.c b/bpf/nat64.c new file mode 100644 index 0000000..f70dfc4 --- /dev/null +++ b/bpf/nat64.c @@ -0,0 +1,800 @@ +// SPDX-License-Identifier: (MIT OR GPL-2.0) +// +// A stateless IPv6/IPv4 translator (RFC 7915) on the tc egress of sixup-nat64. Each IPv6 source is +// given an IPv4 address of its own from a private pool, so no ports are touched: the stateful part +// of NAT64, the sharing of the line's IPv4 address, is left to conntrack and the source NAT on the +// way out, as for any other IPv4 host behind the router. A translated packet is handed back to +// the same device's ingress and enters the stack as a new packet of the other family. +// +// TCP, UDP and ICMP are translated, ICMP errors with the packet they quote, and fragments one by +// one. Fragments reach the program as they were sent, or as conntrack's reassembly is cut again +// on the way out of the device, no larger than the pool route's MTU leaves room for. Two kinds +// need the whole datagram for their new checksum and go to sixup instead, through the perf buffer +// slow: fragmented ICMP, whose IPv6 checksum covers a length no single fragment shows, and IPv4 +// UDP without a checksum, which IPv6 requires. sixup sends them back translated through the same +// device, marked, and the program hands them to its ingress like the rest. Dropped and counted: IPv4 options, IPv6 extension +// headers other than a fragment header, and ICMP messages RFC 7915 does not translate. +// +// No packet reaches the program with CHECKSUM_COMPLETE: forwarding turns it into CHECKSUM_NONE +// (skb_forward_csum), and the router's own packets are never received ones. So only the checksum +// fields themselves are kept right, and with BPF_F_PSEUDO_HDR for CHECKSUM_PARTIAL. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IP_MF 0x2000 +#define IP_DF 0x4000 +#define IP_OFFSET 0x1fff + +#define IPV6_MIN_MTU 1280 +#define IPV4_MIN_MTU 68 + +#define PROBES 8 // pool slots tried for a new source before giving up + +// Set by the loader +volatile const __u32 pool_base; // the first address of the pool, host order +volatile const __u32 pool_mask; // the pool's size less one, a power of two less one +volatile const __u8 pref6[16]; // the NAT64 prefix, its host bits zero +volatile const __u8 mask6[16]; // ones over the prefix's length +volatile const __u8 v4pos[4]; // where RFC 6052 puts the bytes of an IPv4 address after it +volatile const __u32 slow_mark; // marks the packets sixup sends back translated + +enum { + STAT_OUT, // IPv6 translated to IPv4 + STAT_IN, // IPv4 translated to IPv6 + STAT_DROP_PROTO, // not TCP, UDP or ICMP, or with IPv4 options or IPv6 extension headers + STAT_SLOW, // handed to sixup whole + STAT_DROP_ICMP, // an ICMP message RFC 7915 does not translate, or one too short to + STAT_DROP_POOL, // no free pool address for a new source + STAT_DROP_NOMAP, // for a pool address no source has, or quoting one + STAT_DROP_ERR, // a helper failed + STAT_MAX, +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, STAT_MAX); + __type(key, __u32); + __type(value, __u64); +} stats SEC(".maps"); + +// The two directions of the address mapping, kept in step; sixup removes the pairs conntrack has +// no flow for +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 65536); + __type(key, struct in6_addr); + __type(value, __be32); +} v4_of SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 65536); + __type(key, __be32); + __type(value, struct in6_addr); +} v6_of SEC(".maps"); + +// The packets handed to sixup, each after its length +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} slow SEC(".maps"); + +// The IPv4 UDP datagrams without a checksum whose first fragment went to sixup, so that the rest +// follows it there: the kernel cuts a reassembled datagram again first fragment first +struct frag4 { + __be32 saddr; + __be32 daddr; + __be16 id; + __u16 pad; +}; + +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 1024); + __type(key, struct frag4); + __type(value, __u8); +} slow_frags SEC(".maps"); + +// The IPv6 pseudo-header, which the ICMPv6 checksum covers and the ICMP one does not +struct pseudo6 { + struct in6_addr saddr; + struct in6_addr daddr; + __be32 len; + __be32 nexthdr; +}; + +struct frag6 { + __u8 nexthdr; + __u8 reserved; + __be16 frag_off; // offset in 8-byte units << 3, then two reserved bits and M + __be32 id; +}; + +// The fixed eight bytes of an ICMP or ICMPv6 header +union icmp8 { + __u8 b[8]; + __be16 h[4]; + __be32 w[2]; +}; + +// What an ICMP error quotes, header and all, as the checksum difference sees it +struct quote4 { + union icmp8 icmp; + struct iphdr ip; + __u8 l4[8]; +}; + +// A quoted IPv6 packet may carry a fragment header; without one, the eight bytes after the IPv6 +// header sit where it would, and the quote is 56 bytes long instead of 64 +struct quote6 { + union icmp8 icmp; + struct ipv6hdr ip; + struct frag6 fh; + __u8 l4[8]; +}; + +static __always_inline int count(__u32 stat, int verdict) +{ + __u64 *n = bpf_map_lookup_elem(&stats, &stat); + if (n) + *n += 1; + return verdict; +} + +// to_slow hands the packet to sixup, Ethernet header and all, and takes it from the stack. It is +// stolen rather than shot: a drop would fail the transmission, and the kernel cutting a datagram +// into fragments gives up on the rest at the first failure. +static __always_inline int to_slow(struct __sk_buff *skb) +{ + __u32 len = skb->len; + if (bpf_perf_event_output(skb, &slow, BPF_F_CURRENT_CPU | (__u64)len << 32, &len, sizeof(len))) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + return count(STAT_SLOW, TC_ACT_STOLEN); +} + +static __always_inline __u16 ip_checksum(const struct iphdr *ip) +{ + const __u16 *w = (const __u16 *)ip; + __u32 sum = 0; +#pragma unroll + for (int i = 0; i < (int)sizeof(*ip) / 2; i++) + sum += w[i]; + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + return ~sum; +} + +static __always_inline __u32 hash6(const struct in6_addr *a) +{ + __u32 h = a->s6_addr32[0] * 0x9e3779b1; + h = (h ^ a->s6_addr32[1]) * 0x85ebca6b; + h = (h ^ a->s6_addr32[2]) * 0xc2b2ae35; + h = (h ^ a->s6_addr32[3]) * 0x27d4eb2f; + return h ^ (h >> 15); +} + +static __always_inline int in_pref64(const struct in6_addr *a) +{ +#pragma unroll + for (int i = 0; i < 16; i++) + if ((a->s6_addr[i] & mask6[i]) != pref6[i]) + return 0; + return 1; +} + +// embed4 is the IPv6 address of an IPv4 one behind the NAT64 prefix (RFC 6052 section 2.2). +static __always_inline void embed4(struct in6_addr *a, __be32 v4) +{ + const __u8 *b = (const __u8 *)&v4; +#pragma unroll + for (int i = 0; i < 16; i++) + a->s6_addr[i] = pref6[i]; +#pragma unroll + for (int i = 0; i < 4; i++) + a->s6_addr[v4pos[i] & 15] = b[i]; +} + +// extract4 is the IPv4 address an address in the NAT64 prefix carries. +static __always_inline __be32 extract4(const struct in6_addr *a) +{ + __be32 v4; + __u8 *b = (__u8 *)&v4; +#pragma unroll + for (int i = 0; i < 4; i++) + b[i] = a->s6_addr[v4pos[i] & 15]; + return v4; +} + +// map4 finds the pool address of an IPv6 source, giving it a free one the first time it is seen. +static __always_inline int map4(const struct in6_addr *src, __be32 *out) +{ + __be32 *v = bpf_map_lookup_elem(&v4_of, src); + if (v) { + *out = *v; + return 0; + } + __u32 h = hash6(src); +#pragma unroll + for (int i = 0; i < PROBES; i++) { + __be32 a = bpf_htonl(pool_base + ((h + i) & pool_mask)); + if (bpf_map_update_elem(&v6_of, &a, src, BPF_NOEXIST)) + continue; + if (!bpf_map_update_elem(&v4_of, src, &a, BPF_NOEXIST)) { + *out = a; + return 0; + } + // Another CPU mapped the same source meanwhile + bpf_map_delete_elem(&v6_of, &a); + v = bpf_map_lookup_elem(&v4_of, src); + if (!v) + return -1; + *out = *v; + return 0; + } + return -1; +} + +// l4_csum_offset is where the checksum sits in a TCP or UDP header. +static __always_inline int l4_csum_offset(__u8 proto) +{ + return proto == IPPROTO_TCP ? 16 : 6; +} + +// The pointer of a Parameter Problem, from one header layout to the other (RFC 7915 figures 3 and 6) +static __always_inline int pointer_4to6(__u8 p) +{ + if (p <= 1) + return p; + if (p <= 3) + return 4; // Total Length: Payload Length + if (p == 8) + return 7; // TTL: Hop Limit + if (p == 9) + return 6; // Protocol: Next Header + if (p >= 12 && p <= 15) + return 8; + if (p >= 16 && p <= 19) + return 24; + return -1; +} + +static __always_inline int pointer_6to4(__u32 p) +{ + if (p <= 1) + return p; + if (p == 4 || p == 5) + return 2; + if (p == 6) + return 9; + if (p == 7) + return 8; + if (p >= 8 && p <= 23) + return 12; + if (p >= 24 && p <= 39) + return 16; + return -1; +} + +// icmp_6to4 turns an ICMPv6 header into its ICMP counterpart (RFC 7915 section 5.2). It returns 0 +// for an echo, 1 for an error quoting a packet, and -1 for a message that is not translated. +static __always_inline int icmp_6to4(union icmp8 *h) +{ + __u8 code = h->b[1]; + __u32 v = bpf_ntohl(h->w[1]); + int p; + switch (h->b[0]) { + case 128: // Echo Request + h->b[0] = 8; + return 0; + case 129: // Echo Reply + h->b[0] = 0; + return 0; + case 1: // Destination Unreachable + h->b[0] = 3; + switch (code) { + case 0: // no route + case 2: // beyond the scope of the source + case 3: // address unreachable + h->b[1] = 1; + break; + case 1: // administratively prohibited + h->b[1] = 10; + break; + case 4: // port unreachable + h->b[1] = 3; + break; + default: + return -1; + } + h->w[1] = 0; + return 1; + case 2: // Packet Too Big: Fragmentation Needed, for the 20 bytes the IPv4 header saves + if (v < IPV4_MIN_MTU + 20) + v = IPV4_MIN_MTU + 20; + v -= 20; + if (v > 0xffff) + v = 0xffff; + h->b[0] = 3; + h->b[1] = 4; + h->w[1] = bpf_htonl(v); + return 1; + case 3: // Time Exceeded, the codes the same + h->b[0] = 11; + h->w[1] = 0; + return 1; + case 4: // Parameter Problem + if (code == 1) { // unrecognised Next Header: Protocol Unreachable + h->b[0] = 3; + h->b[1] = 2; + h->w[1] = 0; + return 1; + } + if (code != 0) + return -1; + p = pointer_6to4(v); + if (p < 0) + return -1; + h->b[0] = 12; + h->w[1] = bpf_htonl((__u32)p << 24); + return 1; + } + return -1; +} + +// icmp_4to6 is the other way (RFC 7915 section 4.2). +static __always_inline int icmp_4to6(union icmp8 *h) +{ + __u8 code = h->b[1]; + __u32 v; + int p; + switch (h->b[0]) { + case 8: // Echo Request + h->b[0] = 128; + return 0; + case 0: // Echo Reply + h->b[0] = 129; + return 0; + case 3: // Destination Unreachable + switch (code) { + case 0: // net unreachable + case 1: // host unreachable + case 5: // source route failed + case 6: // destination network unknown + case 7: // destination host unknown + case 8: // source host isolated + case 11: // network unreachable for the TOS + case 12: // host unreachable for the TOS + h->b[0] = 1; + h->b[1] = 0; + h->w[1] = 0; + return 1; + case 9: // network administratively prohibited + case 10: // host administratively prohibited + case 13: // communication administratively prohibited + case 15: // precedence cutoff in effect + h->b[0] = 1; + h->b[1] = 1; + h->w[1] = 0; + return 1; + case 3: // port unreachable + h->b[0] = 1; + h->b[1] = 4; + h->w[1] = 0; + return 1; + case 2: // protocol unreachable: Parameter Problem at the Next Header + h->b[0] = 4; + h->b[1] = 1; + h->w[1] = bpf_htonl(6); + return 1; + case 4: // Fragmentation Needed: Packet Too Big, for the 20 bytes the IPv6 header adds + v = bpf_ntohs(h->h[3]); + v = v ? v + 20 : IPV6_MIN_MTU; + if (v < IPV6_MIN_MTU) + v = IPV6_MIN_MTU; + h->b[0] = 2; + h->b[1] = 0; + h->w[1] = bpf_htonl(v); + return 1; + } + return -1; + case 11: // Time Exceeded, the codes the same + h->b[0] = 3; + h->w[1] = 0; + return 1; + case 12: // Parameter Problem + if (code != 0 && code != 2) + return -1; + p = pointer_4to6(h->b[4]); + if (p < 0) + return -1; + h->b[0] = 4; + h->b[1] = 0; + h->w[1] = bpf_htonl(p); + return 1; + } + return -1; +} + +// icmp6_to_4 translates an ICMPv6 message whose IPv4 header ip is ready but for its length and +// checksum. An error's quote is translated too, back the way the packet it quotes came: from the +// NAT64 prefix to an IPv6 source the pool has an address for. The quote grows or shrinks by the +// difference of the headers, and the checksum is moved by what changed. +static __always_inline int icmp6_to_4(struct __sk_buff *skb, const struct ipv6hdr *ip6, struct iphdr *ip, __u16 len) +{ + const __u32 at = ETH_HLEN + sizeof(*ip6); + union icmp8 old, new; + if (len < sizeof(old) || bpf_skb_load_bytes(skb, at, &old, sizeof(old))) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + __be16 csum = old.h[1]; + old.h[1] = 0; + new = old; + int r = icmp_6to4(&new); + if (r < 0) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + struct pseudo6 ph = { + .saddr = ip6->saddr, + .daddr = ip6->daddr, + .len = bpf_htonl(len), + .nexthdr = bpf_htonl(IPPROTO_ICMPV6), + }; + __s64 seed = bpf_csum_diff((__be32 *)&ph, sizeof(ph), NULL, 0, 0); + ip->protocol = IPPROTO_ICMP; + + if (r == 0) { + ip->check = ip_checksum(ip); + __s64 diff = bpf_csum_diff(old.w, sizeof(old), new.w, sizeof(new), seed); + new.h[1] = csum; + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __be16 ethertype = bpf_htons(ETH_P_IP); + if (bpf_skb_store_bytes(skb, offsetof(struct ethhdr, h_proto), ðertype, sizeof(ethertype), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN, ip, sizeof(*ip), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN + sizeof(*ip), &new, sizeof(new), 0) || + bpf_l4_csum_replace(skb, ETH_HLEN + sizeof(*ip) + 2, 0, diff, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + count(STAT_OUT, 0); + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); + } + + struct quote6 o; + struct quote4 n = {}; + if (len < sizeof(o) - sizeof(o.l4) || bpf_skb_load_bytes(skb, at + sizeof(o.icmp), &o.ip, sizeof(o.ip) + sizeof(o.fh))) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + o.icmp = old; + __u8 proto = o.ip.nexthdr; + __u16 ipl = bpf_ntohs(o.ip.payload_len); + int ifrag = proto == IPPROTO_FRAGMENT, first = 1; + n.ip.frag_off = bpf_htons(IP_DF); + if (ifrag) { + if (len < sizeof(o) || ipl < sizeof(o.fh) || bpf_skb_load_bytes(skb, at + sizeof(o) - sizeof(o.l4), o.l4, sizeof(o.l4))) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + proto = o.fh.nexthdr; + ipl -= sizeof(o.fh); + __u16 fo = bpf_ntohs(o.fh.frag_off); + first = fo >> 3 == 0; + n.ip.frag_off = bpf_htons(fo >> 3 | (fo & 1) << 13); + n.ip.id = bpf_htons(bpf_ntohl(o.fh.id)); + __builtin_memcpy(n.l4, o.l4, sizeof(n.l4)); + } else { + __builtin_memcpy(n.l4, &o.fh, sizeof(n.l4)); + } + if (proto != IPPROTO_TCP && proto != IPPROTO_UDP && proto != IPPROTO_ICMPV6) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + if (!in_pref64(&o.ip.saddr)) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + __be32 *dst = bpf_map_lookup_elem(&v4_of, &o.ip.daddr); + if (!dst) + return count(STAT_DROP_NOMAP, TC_ACT_SHOT); + n.icmp = new; + n.ip.version = 4; + n.ip.ihl = 5; + n.ip.tos = o.ip.priority << 4 | o.ip.flow_lbl[0] >> 4; + n.ip.tot_len = bpf_htons(ipl + sizeof(struct iphdr)); + n.ip.ttl = o.ip.hop_limit; + n.ip.protocol = proto == IPPROTO_ICMPV6 ? IPPROTO_ICMP : proto; + n.ip.saddr = extract4(&o.ip.saddr); + n.ip.daddr = *dst; + n.ip.check = ip_checksum(&n.ip); + // A quoted echo gets its own type; its checksum, like a quoted TCP or UDP one, is left alone + if (first && proto == IPPROTO_ICMPV6 && (n.l4[0] == 128 || n.l4[0] == 129)) + n.l4[0] = n.l4[0] == 128 ? 8 : 0; + __s64 diff = ifrag ? bpf_csum_diff((__be32 *)&o, sizeof(o), (__be32 *)&n, sizeof(n), seed) + : bpf_csum_diff((__be32 *)&o, sizeof(o) - sizeof(o.l4), (__be32 *)&n, sizeof(n), seed); + n.icmp.h[1] = csum; + + // The message shrinks by what the header does, and by the fragment header if the quote has one + int shrink = sizeof(struct ipv6hdr) - sizeof(struct iphdr) + (ifrag ? sizeof(struct frag6) : 0); + ip->tot_len = bpf_htons(len + sizeof(struct iphdr) - shrink); + ip->check = ip_checksum(ip); + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0) || bpf_skb_adjust_room(skb, -shrink, BPF_ADJ_ROOM_NET, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __be16 ethertype = bpf_htons(ETH_P_IP); + if (bpf_skb_store_bytes(skb, offsetof(struct ethhdr, h_proto), ðertype, sizeof(ethertype), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN, ip, sizeof(*ip), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN + sizeof(*ip), &n, sizeof(n), 0) || + bpf_l4_csum_replace(skb, ETH_HLEN + sizeof(*ip) + 2, 0, diff, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + count(STAT_OUT, 0); + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); +} + +static __always_inline int six_to_four(struct __sk_buff *skb) +{ + struct ipv6hdr ip6; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &ip6, sizeof(ip6))) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + // The device's own MLD and router solicitations are no one's loss + if (!in_pref64(&ip6.daddr)) + return TC_ACT_SHOT; + __u8 proto = ip6.nexthdr; + __u16 len = bpf_ntohs(ip6.payload_len); + __be16 frag_off = bpf_htons(IP_DF), id = 0; + int frag = 0, first = 1; + if (proto == IPPROTO_FRAGMENT) { + struct frag6 fh; + if (len < sizeof(fh) || bpf_skb_load_bytes(skb, ETH_HLEN + sizeof(ip6), &fh, sizeof(fh))) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + proto = fh.nexthdr; + if (proto == IPPROTO_ICMPV6) + return to_slow(skb); + len -= sizeof(fh); + __u16 fo = bpf_ntohs(fh.frag_off); + frag = 1; + first = fo >> 3 == 0; + frag_off = bpf_htons(fo >> 3 | (fo & 1) << 13); // DF clear: the pieces may be cut again + id = bpf_htons(bpf_ntohl(fh.id)); + } + if (proto != IPPROTO_TCP && proto != IPPROTO_UDP && proto != IPPROTO_ICMPV6) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + if (len > 0xffff - sizeof(struct iphdr)) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + + __be32 saddr; + if (map4(&ip6.saddr, &saddr)) + return count(STAT_DROP_POOL, TC_ACT_SHOT); + struct iphdr ip = { + .version = 4, + .ihl = 5, + .tos = ip6.priority << 4 | ip6.flow_lbl[0] >> 4, + .tot_len = bpf_htons(len + sizeof(struct iphdr)), + .id = id, + .frag_off = frag_off, + .ttl = ip6.hop_limit, + .protocol = proto, + .saddr = saddr, + .daddr = extract4(&ip6.daddr), + }; + if (proto == IPPROTO_ICMPV6) + return icmp6_to_4(skb, &ip6, &ip, len); + ip.check = ip_checksum(&ip); + + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + if (frag && bpf_skb_adjust_room(skb, -(int)sizeof(struct frag6), BPF_ADJ_ROOM_NET, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __be16 ethertype = bpf_htons(ETH_P_IP); + if (bpf_skb_store_bytes(skb, offsetof(struct ethhdr, h_proto), ðertype, sizeof(ethertype), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN, &ip, sizeof(ip), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + if (first) { + // The pseudo-header keeps its length and protocol; only the addresses change + __s64 diff = bpf_csum_diff((__be32 *)&ip6.saddr, 32, (__be32 *)&ip.saddr, 8, 0); + __u64 flags = BPF_F_PSEUDO_HDR | (proto == IPPROTO_UDP ? BPF_F_MARK_MANGLED_0 : 0); + if (bpf_l4_csum_replace(skb, ETH_HLEN + sizeof(ip) + l4_csum_offset(proto), 0, diff, flags)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + } + count(STAT_OUT, 0); + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); +} + +// icmp4_to_6 is icmp6_to_4 the other way: an error quotes a packet from a pool address to the +// NAT64 prefix. +static __always_inline int icmp4_to_6(struct __sk_buff *skb, const struct iphdr *ip, struct ipv6hdr *ip6, __u16 len) +{ + const __u32 at = ETH_HLEN + sizeof(*ip); + union icmp8 old, new; + if (len < sizeof(old) || bpf_skb_load_bytes(skb, at, &old, sizeof(old))) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + __be16 csum = old.h[1]; + old.h[1] = 0; + new = old; + int r = icmp_4to6(&new); + if (r < 0) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + ip6->nexthdr = IPPROTO_ICMPV6; + + if (r == 0) { + struct pseudo6 ph = { + .saddr = ip6->saddr, + .daddr = ip6->daddr, + .len = bpf_htonl(len), + .nexthdr = bpf_htonl(IPPROTO_ICMPV6), + }; + __s64 seed = bpf_csum_diff(NULL, 0, (__be32 *)&ph, sizeof(ph), 0); + __s64 diff = bpf_csum_diff(old.w, sizeof(old), new.w, sizeof(new), seed); + new.h[1] = csum; + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __be16 ethertype = bpf_htons(ETH_P_IPV6); + if (bpf_skb_store_bytes(skb, offsetof(struct ethhdr, h_proto), ðertype, sizeof(ethertype), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN, ip6, sizeof(*ip6), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN + sizeof(*ip6), &new, sizeof(new), 0) || + bpf_l4_csum_replace(skb, ETH_HLEN + sizeof(*ip6) + 2, 0, diff, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + count(STAT_IN, 0); + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); + } + + struct quote4 o; + struct quote6 n = {}; + if (len < sizeof(o) || bpf_skb_load_bytes(skb, at + sizeof(o.icmp), &o.ip, sizeof(o.ip) + sizeof(o.l4))) + return count(STAT_DROP_ICMP, TC_ACT_SHOT); + o.icmp = old; + __u8 proto = o.ip.protocol; + if (o.ip.ihl != 5 || (proto != IPPROTO_TCP && proto != IPPROTO_UDP && proto != IPPROTO_ICMP)) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + struct in6_addr *src = bpf_map_lookup_elem(&v6_of, &o.ip.saddr); + if (!src) + return count(STAT_DROP_NOMAP, TC_ACT_SHOT); + __u16 fo = bpf_ntohs(o.ip.frag_off); + int ifrag = (fo & (IP_MF | IP_OFFSET)) != 0; + __u8 nexthdr = proto == IPPROTO_ICMP ? IPPROTO_ICMPV6 : proto; + n.icmp = new; + n.ip.version = 6; + n.ip.priority = o.ip.tos >> 4; + n.ip.flow_lbl[0] = (o.ip.tos & 0xf) << 4; + n.ip.payload_len = bpf_htons(bpf_ntohs(o.ip.tot_len) - sizeof(struct iphdr) + (ifrag ? sizeof(n.fh) : 0)); + n.ip.nexthdr = nexthdr; + n.ip.hop_limit = o.ip.ttl; + n.ip.saddr = *src; + embed4(&n.ip.daddr, o.ip.daddr); + __u8 *l4 = (__u8 *)&n.fh; // right after the IPv6 header, unless a fragment header comes first + if (ifrag) { + n.ip.nexthdr = IPPROTO_FRAGMENT; + n.fh.nexthdr = nexthdr; + n.fh.frag_off = bpf_htons((fo & IP_OFFSET) << 3 | !!(fo & IP_MF)); + n.fh.id = bpf_htonl(bpf_ntohs(o.ip.id)); + l4 = n.l4; + } + __builtin_memcpy(l4, o.l4, sizeof(o.l4)); + if (!(fo & IP_OFFSET) && proto == IPPROTO_ICMP && (l4[0] == 8 || l4[0] == 0)) + l4[0] = l4[0] == 8 ? 128 : 129; + + int grow = sizeof(struct ipv6hdr) - sizeof(struct iphdr) + (ifrag ? sizeof(struct frag6) : 0); + len += grow; + ip6->payload_len = bpf_htons(len); + struct pseudo6 ph = { + .saddr = ip6->saddr, + .daddr = ip6->daddr, + .len = bpf_htonl(len), + .nexthdr = bpf_htonl(IPPROTO_ICMPV6), + }; + __s64 seed = bpf_csum_diff(NULL, 0, (__be32 *)&ph, sizeof(ph), 0); + __s64 diff = ifrag ? bpf_csum_diff((__be32 *)&o, sizeof(o), (__be32 *)&n, sizeof(n), seed) + : bpf_csum_diff((__be32 *)&o, sizeof(o), (__be32 *)&n, sizeof(n) - sizeof(n.l4), seed); + n.icmp.h[1] = csum; + + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0) || bpf_skb_adjust_room(skb, grow, BPF_ADJ_ROOM_NET, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __be16 ethertype = bpf_htons(ETH_P_IPV6); + if (bpf_skb_store_bytes(skb, offsetof(struct ethhdr, h_proto), ðertype, sizeof(ethertype), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN, ip6, sizeof(*ip6), 0) || + (ifrag ? bpf_skb_store_bytes(skb, ETH_HLEN + sizeof(*ip6), &n, sizeof(n), 0) + : bpf_skb_store_bytes(skb, ETH_HLEN + sizeof(*ip6), &n, sizeof(n) - sizeof(n.l4), 0)) || + bpf_l4_csum_replace(skb, ETH_HLEN + sizeof(*ip6) + 2, 0, diff, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + count(STAT_IN, 0); + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); +} + +static __always_inline int four_to_six(struct __sk_buff *skb) +{ + struct iphdr ip; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &ip, sizeof(ip))) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + if ((bpf_ntohl(ip.daddr) & ~pool_mask) != pool_base) + return TC_ACT_SHOT; // nothing of the pool's + if (ip.ihl != 5) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + __u8 proto = ip.protocol; + if (proto != IPPROTO_TCP && proto != IPPROTO_UDP && proto != IPPROTO_ICMP) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + __u16 tot = bpf_ntohs(ip.tot_len); + if (tot < sizeof(ip)) + return count(STAT_DROP_PROTO, TC_ACT_SHOT); + __u16 len = tot - sizeof(ip); + __u16 fo = bpf_ntohs(ip.frag_off); + int frag = (fo & (IP_MF | IP_OFFSET)) != 0, first = (fo & IP_OFFSET) == 0; + if (frag && proto == IPPROTO_ICMP) + return to_slow(skb); + if (proto == IPPROTO_UDP) { + struct frag4 k = {.saddr = ip.saddr, .daddr = ip.daddr, .id = ip.id}; + if (first) { + __be16 check; + if (bpf_skb_load_bytes(skb, ETH_HLEN + sizeof(ip) + 6, &check, sizeof(check))) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + if (!check) { + __u8 one = 1; + if (frag) + bpf_map_update_elem(&slow_frags, &k, &one, BPF_ANY); + return to_slow(skb); + } + } else if (bpf_map_lookup_elem(&slow_frags, &k)) { + return to_slow(skb); + } + } + + struct in6_addr *daddr = bpf_map_lookup_elem(&v6_of, &ip.daddr); + if (!daddr) + return count(STAT_DROP_NOMAP, TC_ACT_SHOT); + struct ipv6hdr ip6 = { + .version = 6, + .priority = ip.tos >> 4, + .flow_lbl = {(ip.tos & 0xf) << 4}, + .payload_len = bpf_htons(len), + .nexthdr = proto, + .hop_limit = ip.ttl, + .daddr = *daddr, + }; + embed4(&ip6.saddr, ip.saddr); + if (proto == IPPROTO_ICMP) + return icmp4_to_6(skb, &ip, &ip6, len); + + struct frag6 fh = { + .nexthdr = proto, + .frag_off = bpf_htons((fo & IP_OFFSET) << 3 | !!(fo & IP_MF)), + .id = bpf_htonl(bpf_ntohs(ip.id)), + }; + if (frag) { + ip6.nexthdr = IPPROTO_FRAGMENT; + ip6.payload_len = bpf_htons(len + sizeof(fh)); + } + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + if (frag && bpf_skb_adjust_room(skb, sizeof(fh), BPF_ADJ_ROOM_NET, 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __be16 ethertype = bpf_htons(ETH_P_IPV6); + if (bpf_skb_store_bytes(skb, offsetof(struct ethhdr, h_proto), ðertype, sizeof(ethertype), 0) || + bpf_skb_store_bytes(skb, ETH_HLEN, &ip6, sizeof(ip6), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + __u32 l4 = ETH_HLEN + sizeof(ip6); + if (frag) { + if (bpf_skb_store_bytes(skb, l4, &fh, sizeof(fh), 0)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + l4 += sizeof(fh); + } + if (first) { + __s64 diff = bpf_csum_diff((__be32 *)&ip.saddr, 8, (__be32 *)&ip6.saddr, 32, 0); + __u64 flags = BPF_F_PSEUDO_HDR | (proto == IPPROTO_UDP ? BPF_F_MARK_MANGLED_0 : 0); + if (bpf_l4_csum_replace(skb, l4 + l4_csum_offset(proto), 0, diff, flags)) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + } + count(STAT_IN, 0); + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); +} + +SEC("tc") +int nat64(struct __sk_buff *skb) +{ + // What sixup's slow path translated comes in like what the program translates + if (skb->mark == slow_mark) { + skb->mark = 0; + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); + } + if (skb->protocol == bpf_htons(ETH_P_IPV6)) + return six_to_four(skb); + if (skb->protocol == bpf_htons(ETH_P_IP)) + return four_to_six(skb); + return TC_ACT_SHOT; +} + +char LICENSE[] SEC("license") = "Dual MIT/GPL"; diff --git a/firewall.go b/firewall.go index 6f383f8..11dd4a7 100644 --- a/firewall.go +++ b/firewall.go @@ -70,7 +70,7 @@ type firewall struct { lans []string // the interfaces sixup serves inbound bool // filter unsolicited traffic from the WAN source bool // refuse LAN sources outside ours (-source-filter) - nat64 bool // Jool translates behind joolOutside + nat64 bool // a translator sits behind joolOutside holeIn chan []portMapping // PCP's pinholes, the latest set replacing the last delegIn chan []netip.Prefix // the prefixes delegated to downstream routers, likewise holes *nftables.Set diff --git a/flags.go b/flags.go index 27bdaa8..1c8020b 100644 --- a/flags.go +++ b/flags.go @@ -54,7 +54,7 @@ var ( tunMTU = flag.Int("tunnel-mtu", 0, "MTU of the tunnel device, 0 means the WAN interface MTU minus 40") tunMetric4 = flag.Uint("tunnel-route4-metric", 4096, "metric of the IPv4 default route added through the tunnel device once its IPv4 is known; the metric is high on purpose, so an existing IPv4 default route keeps winning and the tunnel only takes over when there is none. 0 adds no route") tunCap = flag.Bool("tunnel-capture", true, "when running as a daemon and DHCPv6 sends no tunnel option, capture tunnel traffic to infer the parameters") - nat64 = flag.String("nat64", "off", "which NAT64 implementation to configure: jool uses the Jool kernel module (https://jool.mx, 4.1 or later; when it is not loaded, sixup runs /sbin/modprobe jool once), run in a network namespace of its own behind the veth sixup-nat64, translating -nat64-prefix for the LAN and this router alike. The namespace exists, and the prefix is advertised in the RA unless -ra-pref64 is off, only while the module is loaded, which is checked every minute. The IPv4 output leaves by this host's IPv4 route, through the tunnel's source NAT or any other, so a MAP-E line's ports have one allocator. A firewall has to let traffic through sixup-nat64: the LAN's, forwarded, and the answers to this router's own, arriving on it. off configures none. DNS64 is not part of this and is left to a resolver of your choosing") + nat64 = flag.String("nat64", "off", "which NAT64 implementation to configure: jool uses the Jool kernel module (https://jool.mx, 4.1 or later; when it is not loaded, sixup runs /sbin/modprobe jool once), run in a network namespace of its own behind the veth sixup-nat64, translating -nat64-prefix for the LAN and this router alike. The namespace exists, and the prefix is advertised in the RA unless -ra-pref64 is off, only while the module is loaded, which is checked every minute. The IPv4 output leaves by this host's IPv4 route, through the tunnel's source NAT or any other, so a MAP-E line's ports have one allocator. A firewall has to let traffic through sixup-nat64: the LAN's, forwarded, and the answers to this router's own, arriving on it. bpf is an experimental translator of sixup's own that needs no module but Linux 6.6 or later: it numbers IPv6 sources from 198.18.0.0/15 and leaves the rest to the source NAT. off configures none. DNS64 is not part of this and is left to a resolver of your choosing") nat64Prefix = flag.String("nat64-prefix", nat64WKP.String(), "prefix the NAT64 translates (RFC 6052), advertised in the RA while it does (see -ra-pref64). Reaching private IPv4 addresses needs a network-specific prefix such as fd00:64::/96, and one carved from the delegated prefix would change with it") joolIPv4 = flag.String("jool-ipv4", "192.168.255.254/31", "the IPv4 /31 between this namespace and Jool's: the lower address is the gateway on this side, the upper one Jool's pool4. It must not overlap any address or route of this host") unsolicited = flag.String("unsolicited", "request", "unsolicited IPv6 from the WAN to the LAN (RFC 6092): request lets in only what the LAN started, endpoints that recently sent out, ports opened with PCP (UDP 5351), and the ICMPv6 and IPsec that must pass; deny does the same without PCP; allow lets it all in. PCP, and NAT-PMP for programs that know only it, also forward ports of the tunnel's IPv4 while sixup does its NAT. The RFC 7084 border for ULA and LAN prefixes holds in every mode") diff --git a/go.mod b/go.mod index 75fcc4a..4419ab3 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( ) require ( + github.com/cilium/ebpf v0.22.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/nftables v0.3.0 // indirect github.com/josharian/native v1.1.0 // indirect diff --git a/go.sum b/go.sum index fbf178e..c778bbc 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ +github.com/cilium/ebpf v0.22.0 h1:v2ktp0roffpMOj2MMf3idtCQZOsAoC4BJbAJN+ke2bY= +github.com/cilium/ebpf v0.22.0/go.mod h1:CDzZbe2hC5JjlDC+CY3KFCzlYwN4gbxppYM+Z10bQt4= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= @@ -21,10 +24,12 @@ github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA= github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= diff --git a/main.go b/main.go index cb9029f..69175ad 100644 --- a/main.go +++ b/main.go @@ -135,8 +135,8 @@ func main() { if *tunNAT != "auto" && *tunNAT != "off" { fatalf("-tunnel-nat must be auto / off") } - if *nat64 != "jool" && *nat64 != "off" { - fatalf("-nat64 must be jool / off") + if *nat64 != "jool" && *nat64 != "bpf" && *nat64 != "off" { + fatalf("-nat64 must be jool / bpf / off") } joolLink, err := netip.ParsePrefix(*joolIPv4) if err != nil || !joolLink.Addr().Is4() || joolLink.Bits() != 31 { @@ -281,6 +281,10 @@ func main() { go nat.run(ctx, store.Subscribe()) } } + var names []string + for _, l := range lanDefs { + names = append(names, l.iface) + } if *nat64 == "jool" { // Jool's IPv4 output comes back through the veth and is forwarded out of this namespace if !*noSysctl { @@ -290,12 +294,17 @@ func main() { } go (&joolManager{prefix: nat64Pfx, link: joolLink, store: store}).run(ctx) } - var names []string - for _, l := range lanDefs { - names = append(names, l.iface) + if *nat64 == "bpf" { + // The translated IPv4 comes in on the dummy device and is forwarded out of it + if !*noSysctl { + if err := sysctlWrite("/proc/sys/net/ipv4/ip_forward", "1"); err != nil { + warnf("[sysctl] ipv4/ip_forward=1 failed: %v", err) + } + } + go (&bpfNAT64{prefix: nat64Pfx, lans: names, store: store}).run(ctx) } // The border of RFC 7084 is kept in every mode; allow only lets unsolicited traffic in - fw := &firewall{wan: *wan, lans: names, inbound: *unsolicited != "allow", source: *srcFilter, nat64: *nat64 == "jool", holeIn: make(chan []portMapping, 1), delegIn: make(chan []netip.Prefix, 1)} + fw := &firewall{wan: *wan, lans: names, inbound: *unsolicited != "allow", source: *srcFilter, nat64: *nat64 != "off", holeIn: make(chan []portMapping, 1), delegIn: make(chan []netip.Prefix, 1)} go fw.run(ctx, store.Subscribe()) if *unsolicited != "deny" { p := &pcpServer{lans: names, nat: nat} diff --git a/nat64_bpfeb.go b/nat64_bpfeb.go new file mode 100644 index 0000000..643f900 --- /dev/null +++ b/nat64_bpfeb.go @@ -0,0 +1,192 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build mips || mips64 || ppc64 || s390x + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "structs" + + "github.com/cilium/ebpf" +) + +type nat64Frag4 struct { + _ structs.HostLayout + Saddr uint32 + Daddr uint32 + Id uint16 + Pad uint16 +} + +type nat64In6Addr struct { + _ structs.HostLayout + In6U struct { + _ structs.HostLayout + U6Addr8 [16]uint8 + } +} + +// Names of all BPF objects in the ELF. +// +// Used for safe lookups in a Collection or CollectionSpec. +const ( + nat64MapSlow = "slow" + nat64MapSlowFrags = "slow_frags" + nat64MapStats = "stats" + nat64MapV4Of = "v4_of" + nat64MapV6Of = "v6_of" + nat64ProgNat64 = "nat64" + nat64VarMask6 = "mask6" + nat64VarPoolBase = "pool_base" + nat64VarPoolMask = "pool_mask" + nat64VarPref6 = "pref6" + nat64VarSlowMark = "slow_mark" + nat64VarV4pos = "v4pos" +) + +// loadNat64 returns the embedded CollectionSpec for nat64. +func loadNat64() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_Nat64Bytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load nat64: %w", err) + } + + return spec, err +} + +// loadNat64Objects loads nat64 and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *nat64Objects +// *nat64Programs +// *nat64Maps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadNat64Objects(obj any, opts *ebpf.CollectionOptions) error { + spec, err := loadNat64() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// nat64Specs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type nat64Specs struct { + nat64ProgramSpecs + nat64MapSpecs + nat64VariableSpecs +} + +// nat64ProgramSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type nat64ProgramSpecs struct { + Nat64 *ebpf.ProgramSpec `ebpf:"nat64"` +} + +// nat64MapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type nat64MapSpecs struct { + Slow *ebpf.MapSpec `ebpf:"slow"` + SlowFrags *ebpf.MapSpec `ebpf:"slow_frags"` + Stats *ebpf.MapSpec `ebpf:"stats"` + V4Of *ebpf.MapSpec `ebpf:"v4_of"` + V6Of *ebpf.MapSpec `ebpf:"v6_of"` +} + +// nat64VariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type nat64VariableSpecs struct { + Mask6 *ebpf.VariableSpec `ebpf:"mask6"` + PoolBase *ebpf.VariableSpec `ebpf:"pool_base"` + PoolMask *ebpf.VariableSpec `ebpf:"pool_mask"` + Pref6 *ebpf.VariableSpec `ebpf:"pref6"` + SlowMark *ebpf.VariableSpec `ebpf:"slow_mark"` + V4pos *ebpf.VariableSpec `ebpf:"v4pos"` +} + +// nat64Objects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. +type nat64Objects struct { + nat64Programs + nat64Maps + nat64Variables +} + +func (o *nat64Objects) Close() error { + return _Nat64Close( + &o.nat64Programs, + &o.nat64Maps, + ) +} + +// nat64Maps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. +type nat64Maps struct { + Slow *ebpf.Map `ebpf:"slow"` + SlowFrags *ebpf.Map `ebpf:"slow_frags"` + Stats *ebpf.Map `ebpf:"stats"` + V4Of *ebpf.Map `ebpf:"v4_of"` + V6Of *ebpf.Map `ebpf:"v6_of"` +} + +func (m *nat64Maps) Close() error { + return _Nat64Close( + m.Slow, + m.SlowFrags, + m.Stats, + m.V4Of, + m.V6Of, + ) +} + +// nat64Variables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. +type nat64Variables struct { + Mask6 *ebpf.Variable `ebpf:"mask6"` + PoolBase *ebpf.Variable `ebpf:"pool_base"` + PoolMask *ebpf.Variable `ebpf:"pool_mask"` + Pref6 *ebpf.Variable `ebpf:"pref6"` + SlowMark *ebpf.Variable `ebpf:"slow_mark"` + V4pos *ebpf.Variable `ebpf:"v4pos"` +} + +// nat64Programs contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. +type nat64Programs struct { + Nat64 *ebpf.Program `ebpf:"nat64"` +} + +func (p *nat64Programs) Close() error { + return _Nat64Close( + p.Nat64, + ) +} + +func _Nat64Close(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed nat64_bpfeb.o +var _Nat64Bytes []byte diff --git a/nat64_bpfeb.o b/nat64_bpfeb.o new file mode 100644 index 0000000000000000000000000000000000000000..4c7b60f12b16a5191ec3921250d4a7527c4ad101 GIT binary patch literal 70672 zcmch=3v^u9b*6dnE<7j{Un2OBbt#cl0RqV)2tp8Fx)fiaM3EJVG!feh3Iss{Ly#x} zAPJgwTu$3=w&R{u+)j*moOIceEjZ(8SKRK9y~eXplVmbX;&@eOsaJE>IGvhao(W=W zrfXuS8;&K{{QI1JF3#NqD3bEHmI~*7cki>$KJUl9MSbkx(L1&R$-8CzfdW`Pe)K*p9 zz?aHt;+Z!3t5lt*?W9|bT;HQV9KQnbtB9|$yv~Fb*SVyue>JN7x1=Zh^B3`Nh2!5e z#d~=F%Kw?-BmEyzDD!b?8{JjvQd>DrVP6ORMS3QvJf-ida*Jnf*uR&aH&?HAmcK(+ zmzJ8gacO5+TkKnIC&_2VAtYkj2uZ5?l^sRyL5 zTi62W>v}@^e@OZZq+hM=w3Rz4ypQy?KGOfXj<*y=j1SV^LE!@FYdO;YD#;f}U-t*( zze~sGdbmgawLa4SBI%?2>vXuY{EUu&clm2M(*JFeFOa_OuSmaJ$A90NB(zHcpvF&eWX94<1JN7bhxv;N5{WA|5}dp&yswB^dHgj+sb=&{C6*Zt&j9S zq~k4B%{ts!eon`~JAExj`tK(>k0CmKGsh&S+(zWIG zFzL3EMWjphsfNE$)+7DKjLy&U_mCgmeqeWvw)^4oAqr=nqQAD>o-3bBs}^7d%5N3j z@9(0&N)5bL*7bkR%ts#82j+Lvcp*Q|{bl9_(#E{)Yk9A2XR`io>eivExxK8$JidwI zIqfY?EZSRIwjZax>G`Ne{p@EH@A{Wb`w!>u@iOk~{)K#AfZ9^Di%*a~+MRBvwwin0=pC&o_;{gi6Hz-6suclD+tGvxGw~rY2t4j8h);rx4&&lTl_N$-$pA;YY{68E! zRo3tOFR!YCu~YRO3hzIjI!O9XyP)=>9A)eeibuvyXH7AEj_c_fm{UQ9T~L zW#?P;V?DoKT2t;JJ=ELd6mq)M&-FNo^f%CbgdT@_z5&OGG4Jf7aE8Y}?g!Q5ua$Mm zc0M}nk(R0*|8iOPm+@&cpZ9xuK2ZZ-HuDTvf536N==W3;gEuKJo=#`!+{(B&D}xp%y% z6Hk}>ea4Ff@wS`#;eAN+jPXY`bAHCz&lz1C7?0AlFIN>aDebqq_60LV+kS8=O83iU zo}VI{*Dts}EZOmy^`5xtYkSW6Q0eiL`%Cph^iy5G7$_A_MFqFbB+g`f8P1O zOuFyCq;)%pqrLq-vE6rZIy4{X`)E%klF#VT z6z$L>QgP_d(uLi(Ts7^cdoZqOoT^(HS3b!RQv0c@QI(GppYrIhDi3<b>3yX23+?DF(lcC9jdrQyarvi_UI)Yxgfy-;a{5h3UytLQ{yQhV zN0I-=h0j*?w*sdwbE2K1p`i&xh%c6BHlqNLyq(v`%o!Lsem%(d`1|@_RZx zpZ=>v{dpnEC&>o}Kl+<_1Zir|S!H{juwByAQkmni)U) zW2~oF3y9Za?DhAWbl~snWZ+bW{+7!FUn^(n>Rf{UD)0CS9jfbJDNoVW0_o_OvOa8i z-1JO*ZjUihAM0lHDl1r@m4uyHjvwLp5j#)o%2R4xFwgON8~yYLl;X`a{l$6oqZEQY z`n%wG{$9t)?PMA4K(9N_Prv8!QO^^i9PWF3JgLk&NW`BFmxBtim~YzdIGByXEr6D2=PtZyCR<-{R*7D{MbTzkP}{ zG5$m-JQ=Oj>lV>&kJ@&6ZKqksaC`McIj!UJiSqsU%CsKWmoi70Ygwd6|7IF9_ENS{zHswZ~t!;zec_N?;Sg0T&tjQzUm(G3iQJT$2azi=GR*kA7PHe zX%4d->hT70J-#5+6PwDoiN+->EKmzyw9o>gsghI4K6e-RXja~)LZ}t7rWhCYJ zbiNo>`VQ;S`Qn56`@3Wh{{9YyTrSm{pDgQs$6P1p|CRA_CDbG6Uq3-u&w903zv~7s z%InPs*xx^>|GMhn|HsLHtk=Fvp3#kUx)^RN|2c)%cA9jWc^s>Y z>-FYa=U+3=Z@4^n!k?1X=lW-VgZ3~lH+t|-uZQ~Myl#M9e7t_2*A1PT71*p75{Bjc z-r}44_U|RFjr?cLdH_u8R$^MG(t04H!`IBZ;8G`_E4)_zGTjI3@d5ks*O;|MuwHLK zPVHTRzru3e{=i>l*7Fef4dVWtkJ+mIa z!Cz-K=L%mb{}0TxURRL+E6oS^T;cVy?%yq3ek7yMC7lb@zrkkF53=?gq1iVIGrCl; zZ!p+0&i7$Qmv3O_d84pMyr>gTx99te7YX8>&Ys+ZZ-2b_V2Hize$S1k6ZmO zl#BZQ3Hpn2H2z0$&h$Ov@g3%TY9=g?;=XBboo0P=)hBJ$GTv$Wg{>#PAHjWHuHt?a z_f0?OFy+VjX&L8!eeDHaC;0k&i}y>9mwDV*_I%-RRHq+Vsn;JW0_Sx9pm81NQR$Ut zzZYTt`1m%hkNL%nBOHH)eGb_Cn?`QWG5S~Wcwo};uwT&a{zj_zF&vx`s?Ks zT`iE_5ZxE`-pliyZD)$-@w@gDe7_szG)s24e67AO)@^x_PMX{ChWprO{F?UjQTl7! z|K0nGlb8!gZr35`H*;D)PJa>4*MVI}(>amuiJV_rzr9TwVqAH9vH5kS`l%kDYt&Em zJkNIM9Q`e_W4O8k^Y&k8_UNyQoHFg??Um7=Q9o%yd!_atKKM?Oc$&ZRGyb;-$$K1bZ?JP*o~-0sU;JUxDxO$hGWaaBB*4bJOG z-~F&3)t@vs>A9>=mt0>Dv7h+qaf*+6f0070*G#|R@$9N;cU4?J6?Q$w?Jwx>_CDux ziJ2!nb)4C`WG+IBd)TB|Ev9+f7~G%)t@ zcnN>>I)n8e;<^4$w@?Zwhc*iBdDbcIjXjSqzlD9wdre1=+(_p!N=J+vo*XyqdXnp# z-ly9`@%l}@(4;v=%=vzZ^E@+8USoT_zn1UkVW0kwKcaL*|6Q!Vbj$v7@}bWyLgoDs zEoz;WBmrzmgTj%~Yqi1AUcFc&ND{zuLGo$&XM$zSvrkHV)=Kfm|P zwLic*Y6I~%xZTtJPyX=F&*Oef-v6U-{;M6h-!AVz_2v(^nsF#iBK?{9xC#4o@5j0= zt-1Ldd|$Rpyoahka(wE4!rbrE-r#^pw|2eGOp`>k_q*N~!9J`+dN+Og%ZFAtUq}1- zFx{BZvlALq+N{H*Uv}L)(@olU+d==)`;B+=gZo?6V)ku+oJs3s)XKE_0bOAora!{Z zIQFYra>~=<65fNvW$a_M8FdVuzS1pmVYV6r%kdrI6#%{TGOl z-s2R)PLB>>Gta@tzh>T(vhSbOc_dXUFwV@<1>!$X;a&Sp-RgpUbh9mAyI;PfPg#R` zuI17O-YQFVmY{E+$O?b}JVbq?V+kL8;Kb&~4aJ(HH9^n0vuX)}+?<;s7 zid;Hq^pAI%(C!!PI(Oi8^FG_k43=D&Z?1j7l*6e6>6gpHFPM60W;@qDVCF3=ckV|h zhUMW8m~yu~^GhTR_Cry7-_9@1`;460+x%NwnBTbnT=ld(+N^5_>FhW9Gh8oL@03sX z%VnuQv%=_Kg?)UY&CGA;^vY$N2kLecUXS7Z@wkciNfXfX)!(K|9{*@OP7~i}f1>&8 z{~>)8w(c*of7x+mhQ|+XpA{|a2gRk2eNW!Lf7s0H47>-ICi%Phv)DL;_aetB{(bsY z`3v+b`2c=Z{Q~`Z=>hzz{ssEA^8x&-`33r=ck=hyzG{Dge(4cmF~4ZO#i#BUHQ%Ej zR#JjI&eS~kIP;TlQT!V9lN^O7Q~Gx??RI=K>!0!*$>?LpxtqR5yU(NjzPsLknDH(_ z7BSwLc{)lIq4yQ^j40OAz1BaT&+oM#*7J>s?~C~MJ&hG|J^pjZ*AC8?9(nD#mzmF0 z#mQ*ZBP1989fe+r?sNV;F7Jr&Zyn1!R%Y}%qL$g7C&l@G@LVp)cbu#vU0(;b9`W3* zQm^;l!@RYMETSHlQ@HT{P~GqQTEX*Oo>%EPmo6{%?;*}_-en(`B(GKFOsx*7Upz}! z&igdFJ#qT)<$anun61KlEVRz9K=>%aN`w`3ALaL_2yMS^bl$(vD;em~dtcQEHzNG_ z$+*fROtW8f4*Mk7#t+`#u;;d?bpOK3`BC+6p;mp)BW+c6%7ooCzExB-kT3M9`WWw{ z;K$$54cOQ9d(PPLu5xIeLo0jE*=yQkRW;VjvlOomdN>z;OU5&L#a zo2S3Ld&=kw=$x{LRFc%A>Q=kaT_ zzHNuSH&ceW{9|%{ljE&@zmv<4-jhE<7BJ45aiY!a3o%~!bibMV4W1+EeW=5SguF+v z-e)-adR}z+n9xrM);mb6Z{~5*Fdy5l?7rYslyMAAV9N7kd9mkFV&*df#(1O$ree%v+veTTdLH_FJta;d*(9^@pi1{H|y> z*%^4FT%_>&8|HZ}!UFSc4)YXpdfl5__Y$}@;n}AziB?x>9HM} z&mQo+&Af+he`n>`Nt1u?w!dRP=yz@&>BG)C3V#mof3=Vuv97uAb=OJky%&ft-~S5U z=NA369`SWMGVKR`JWAGradm@jKlgYK?DJY*d+{*8=l-^!F4x@6)7-Z)WJP zukf7nyL2`DVgCK=8uK26eI9x;s@DbW_8jVzN7^`NH-7nezt&xMM>eloupQnK`M=2R z(0LEO;xt)y+Rp>F!=K%u_{i^@6e7Pl3eld66kav{df@L7b2=6FIR-PHcbIkWwNIPp ziyPD8`G#&)ICt=1WZ~xx~xL4TNX&4))jQV`B@#mgZ^Uuw0Qbj z|JBY4UBXDeiu7^*`Y$M4Abl-I`pZbZK>F2mAL)ON!u!a-)<^of-y;1Q9e-!}w{(1- zk4~BTpnnHI%aOkJdx7+8b^JE-JN5fWU+cfxspfRN`QO3oabSV`)B0V#x=iW*y+HaV zA8qEnqWef+>m&VHPG9$Hl>hG#FOa^LBmMtG@&(dwp!+EQFHm?N>1%zYe@Dlg{~i2) zr{mvU{`&a<(*G-xFOa@o$0GeN>iF;8{QzfJn6|0O!yS^ixe|L*j)9O?fN$rngp zj{`_wzps8D`PceL|CUa7{&(>Ia*_1)`$I_onr(Vrt+4arDW4GM`)wxuX1)8wxcGt@r!6$j_y97_rZFj&uB(P9s0wZ9<&4 z>w4$(y{o>yZ^Ni9m*$w!$9s_aW8V{J{Xoy&H~B4>ZG5Do`)>>4yU$1L^PIl@#;#pQ z++~;hX|NAqenZ>C^N-ice?=CsE}`+S0_|AO=PS+oUuoQ@as1=lKJ9y9X=Z9?X8!k} zg$0m3599dltk*5}6KS!I%-{5N{0)okBghWc-``)zKAk_{{w!Ir&s}FYpJskUd$;>H z9(RU!^8Rg}&HinkbqllmCg{JQ?a#~2{*4*$*GEW>^3>y(eV&MZc1*`xMt{+7+RSq| zyN?6EO!qDS|Guw#lv1;>&J0~9R9^(ClcW#iM z_}zu>@oeWI_EYbCjP1}l39V!Ge6ZL$7XPlrH%ZUF&wn*agx|k;I>dWG!SCnwI)(2C z|E|NijMiIX-ZR8|Kzg-@@#fQJ{fze-zC{|?4}XWkQ`&yHY}d&&Uemdd-A7>kt=9`U zFG-WWn3vM`n3r_>#iHr}#pCj$_wC+Zz|KA1+k58}>7yQAq0rdT=lJg@7XA1^-fL@F zApU#ZPv(Pm>5+A1_mdlLCN$gj`E%G$hW2R>)4touBa~~5Le%dA*2{wL<rIl}-8hFM6Cacklo2fy@0P-b+VLY(1RN zO~K6|j32&e2`6If^f) z6!U>|6rJv0#+U0v{VJ~4;Y*(z_|o$f`h1qlU=^-E&&g1rZ##ZGihEQZWcMo^XAQ2u z&ev;k{Y}SBnb*~jQ+!hYpE-^WoHQO|9n!OPPWLDx-ShCJy3l(Ogj5$x@%8bgx}eoY z8egglde143?`^IN57*kg`?N*z)A*7tvYW^EyKL(TTx(lg9~;37wPo^|9H%T&J&^Ak z@TJe2_|mnH?_cwE8sRLbu?g318hXd!locgS_UrMrW%V>&->pMxfet-a`oD39Zc@Ad z5x%;N5&nd)J%nX^sm^@3L1|H4C-AjtQF|aGWXHpo@)pCFK2$Gl`1<%#z0g>m#+S-+ z3%+^StmS-e1-tp|04{~mCs>c#6*rd(suQ~QphIcVwU2Ac6I~~9y^F8YxIVzwSzP}T zU*~a6evp0<*MsJoT&TFn*B-8^zd)zL$M;iwoy7HT;Ceg4G`^oT_bWYI)42)sD}7vl znXi+$euJ;mxX$r)7T34H5HB(CXx8rM{JsOy?6 zuDfv!J2iQPZaagnZU-G53$HSNEl|CN6PtgTFG%m&UmA>D$7j|ly_i#wQ?sKl6cdifdHS(ekJJ(0%b=nTW zW(Bj)A(id1ItE;SD7YTJIHCIb_PzY_zF+Qp<;VvP_MQ0U#KFEk+`GKJLtVZ&K78)t z;2F=TUGD0fzcn&?@tjA0)Wo&apt{gM**AD`Vr+PDT%8}hrmpmlTpm>86a5q8YWQMT zU;mjiWBTHl|@>{&H3 zcu}R!PxNu_#?Grtmj^Eo_Fd>trG}B~)Y$06=)mZRx-!y#u`e|?IF9Tam>3wgccBB# z;nA_-iEGIIIjD{I4WAvpcxG@?*>GZ@&)gU|t%mx?hZJrMjP#F>51&zI`X~DJAC&GH zb+JD++=sho`!5WSTvKC%7e*%r`-W4UYGibve?(seH@kwHT~;$iWoA_MC0XygFo+~3 zP*oRF>gw>=U>{}u-1unU_#jF|Ur=!x4O1RNS6Tlo6&*@^T#cW{f2ZN$1X{_&fI6Kz zyH&%M0aZVI){8;9dGCe(vGd-ZJ)SOdf>!Uks;_rsUVImBj15j)9=qtlNZ;7t88|gC zVGMAYwR_Je4)q=CJ96ys%LiY1rFED3sFRAV8%l{x3{8w)9FHA5(bJdcJCb;RS8JDq3P!r`LATGuF`} zU}5z9+(>a?CQmd(V=n5$Cgz8HJxk7$VKXQh=}G2O_E<6 zoSYasGsfPtWRG_wk$Cy|iQ|2TUfy^3rGv*#nCc^+D3RT^q|OetdLQ|SF8h0xulKZ# zKt+oZJ3AC)9VTow#i*Xh^%1zfMR(m+WcpWAqoX6}kK<@Wlx?EZ{o{jN(oh{98X?IJ zQ)g}lx7a2U${=)Y22boyc$+qHkvR8H9$5JPk^L_vOfo3&iO~sB;9GEu`ju@Cq!Ep#ct6<7&AGx~t2!}QM4i-Yi9j{)pD z8UmUeeaLIFemR8^eb5d=A?rv17EERBJNC=en#N5z+UePA)8&Ljthvyrre{}J3A!NH z(|Xe(=gp05R}WowAFcdyq;qVTvlKNP~%!3(Dc&veFUK(|fAHWn%iH|mwn z)abYwShwwX6{(yCaBlA!37wvz8@nCRMn}%va}4aCpc)xD6SNzOKwW}sg}Nq&gCQy$ zDlFZ0sbXj{8wrL)Jwyy6rYTV#E68V79gp2L0HAQUql>OzA4pz0qz%|1fk*ZEX#n=@ z$Y4i)Dl;leXCeuuN`(;&?=+Zm0g_)dCtzuJ@!TL5PMCAdtUK?qGgG&FaXSsz!OB|b zKl9&Q9-rtN8y&r14Dd9K#JHIsc%kp*J9h8{+c9fNZ*XF0aEw-VA(x$Gg|p)m zqgWR?D+G+k+4w`9IQ^XZA_VQnm{ddpk-6@N_o4XO2gK?C0p?$v6SwL^ENx- zZ)jM)tBgiEQPkmO1DB1HfL>D0Tis4pd4)+{qZx1B%L^W4#I|(m+3)Wn-Fjz!fvTBd z^nE)zX%iZ#A9ABejSpTvGuq{)Y$vYYLcL;ou*XU?09Xhn^N=Zq!hLGxGpzigd*K#gxUjUYVIL4)U!4kP5>*2wAFWMY!;qI&3-MhMfUj6QVJ09vaD77}ge9yf<3-d!D$KHg34J?{6 z=b<1lIMUgtYkzDoHPSyY7~YZygFq~;&^V~pZNqmuWruFQ)bXxhy-9nYzL$<1>wD?M zNxgDds1IJz^&jdVv{Ia3C*7l&IZf0~uI*lTsOh2*?fgrtLfdj_M|E}tTW~Zuv$adS za`5DV<9$5`UZ&;bUGpw2#Ghl;m-fBVMq zx_8^<*@ERh(upFr64UyOFVqrjv1`Yjx%<*(Y|Fbm>gv~NrAaA-7YFaMG?=gD(JjLR z7g9)U6RuK0%ORlUr|r;NMmqK4!Kfo$ba76YRD+!m?aH_;olVU{ZFbOrLc2ztjxKDR zig4O`wBH^K{aDO*Q@i7u>DuLO-RdRyAZ4d_40}$mXR!aw;8?H>nqQobj}0JSG*{fU zjg2-UxY;S2_M&USEOp4VJqFPwZx2LF5zpt#=HFcz>&S~P*>!o- zq8+({HKb#kAri8yq5EFAlM!7i7cmOxayor^cw}PuB2M-$45Y5bXpJ-qnKOFOZ~;}_ zMS@)bA0W)1dpMhKUDsI8!I*9|=*cve1f*B#aKp5y;C45Sv>n}O6x%wUr36v-=f*~9 zHOfO*&=K^mvGx9Y2uGxn>oPqB_Vun~k4}P8qlMjKU6hty&{=G4=bRtFh+Wz(vaO;dObJE-7bSLG6ruOL=>-obS+0Mz8GYtnEHG zHTPay?R@ur(fs@uEUtnke$i=3)GzeHZGO)c47b$VsBrJvSlG?iy%V7Y4B8M6jo7Z+ z5rbNmE+BKJDt8qW31isZlaezQ=rXsbcsT9D0KVNG-qkNSU|1#0@%nUdn?6X{w!Kvk zkp~WW10#d|V>|T#nHnA(z}R=8|C)Dt&>Ogn!Sh`I@I`Xl?m^74xt+>JV(`rlTGY=r z2Ynb*m2D3?yYF#nfdhKex+s;sbRdM!A_D_7?Y8I&51w}~59sL-S_YN^?B*3S#JAxv z6Or1k@NL8lpAc;_iwPuR+q7K@KJ4Ngm5wbOZ3ySjsojECX?-)Ei(BE}>QN<2c9OyXR(LcC-$D)ecS+V1pGo6E{JI+FpmuHkX zJV3MCwnMi?F6e$PXLAr09J|vZ2I^0H!*n=5e>Sh9+Hs8PCUi(?mu~&Ke$23R?D)an zBd_2joBPmuYJzvi3~CjNV??K2Ll9FRU)X48@K`1|6%7*9e%)uAF%#t#a!9YG%rlbU z9Fk5eu}Pt=jk~hqQ^{b4GpC{`z3XJhu77l>Pw#Lp3L#ku)^_H;d3N9w`?%gt1j~On z1#=QeCzjBkZ(|<}A8MkO-(_^7BPBd@x@wL-%>h%40=-5=ht#W9xX8bl@>m@qYA~ybAGu-AF))}qg1%_ECEIRMq z*9iqVZU=&UP8w*D&Snicwp;W2-8+Z&$cyc&&EDB|t@|9A%)Zeozw3_Hne(^Nuw=$B z+ihrzGeT#EbS4;dbUte6v_}mX6FT%Xd4=YT4t*L&USA0Ycj{P+pGAs@(iOfQIr9I8|j#g}$cf)`# z%*Boi@3Gu;+KN0R3=R@?yBSVlY0O)1bWyvBp>;lIaZE2Z%uGqz3-xBxnRQXw-uNKB znFlU?UebB*^O9iaOgovM`|$}qprDGn?_uD6#{p*;z#>(T188JO>AuF1(Bc!3f3By8 zG=SfAc&NAE3ob741o^-rYL;f*I-H`(iYnbX=WzB~tU3SCU2l5l&j56rp#@&>7<^RM zD%qXCrB6Zb@g_n4#C#DUez=h=xa;Lw4)2g@hoFzxoF@r-*I;sCj@{4b$LW;g#a5X5 zjE-MW;7+t2v5=%LRyu{@0~xt=(1jz{4ltqP(Q6ENi@~DPYkA)h?!NEu5{EMKNXD!) zbdteB!y4z4Dy~|y1E4BdWUaxuG`p0=UL=4+baFri%4Y|3B!F5C-FcQepw6#d8DUf( z>mQiFl3e?0*5cMty(5R4Z75)SZ=n$iM?^eG1*wJ8r6USDprDx`G(FoBIjZsb$1%_Bvn zHIzwu(FGF?l?!#E&_Pqshj`Gyt~5Mm$nCDK{{@Gi#rL)tOzr-A)H$<5nQ`}7Y(J~D z?T(fj`TUNSL{#eh2FLaLUiU3F-aq7UAG9!PNaC5XUBkWmR&=3}K6nF9VZt6$N0013 zcP>Ci{pceI*J~{^b3JR z9te-&5Fmbpwe$D@ZI-qd<%S!-c|0!wS9MWC$90HE8VN6_tNpH{2;#PRMQE( z7p+L&GW}-2bC}+bjWd5rHTB~CS9-r$>wjJ~eTd%CPB8z4YMP`pk&c%CrD{4u_LB~i zzZvF#RFM<(&hE6}g5jzis`4nMH_JSzDihGJV1KGEsD?S>9+q29h2vM;5b-&E{J(W8 zDgBBmmj8jO{Ba97=Wqmk%VE4zUQuv(BiQ;!@t**f968yWa~SsMt#B%zD$<9&%BaJz zSBW@*9QGM5DTiUND(W!oRZ$#UzOYy2ISg4dnJd--kig*N3X>P z{WXXLss4GFxm4RCVL5o$=;B|WH03~*_(8j?4=zhdl`qx-ju^+ zFY7SbyX7$1%R5Z=3J#OKS%=A9$zifL=P>NmD2HLMCh9Qk)gX>rel?!MkkxcMO!j<- z$zH-?vNz-~{HsYhO!g)nCVOdz$zH}`vNz>0*}KL3H!3oM-`dsOHe4N1ks-_nH6>>J z-(+a!SA8Fn$MAJL_;O*yiUysKy9>YEwJMa$?LkbxLr|u$_YFq*T>awiQ1O#)RbwaS z?+E9w_HkADk=3`dWS%oN^dzkH`&&(Rd;`!FLP?{+F1o|8vaje|-zb z|GaA4jrLg2{?~p>HTq}|^>LOHuLpNKO!9=o2f)1!lRU}%wrV_p#!x?LxUNw(9z}lZ zrv%&j;`*$=W8}6z>x<0ne*APP5m^a969Qz;fBMgpN6c%sGo-0f^B`Uzl{+ixBfOpnK^%r-ON^>%RA6#|84#n z+5fuFs>T!OkBvhnzGkG=ILZ8)YV1XOZ%i})8`bzB($6vfv1&}h|Hgu0TON&fnA4^_ z8jFq`<Z2p?qKbya%tchRu4qQk7ZMtPR@|d(nEh;GRyp3EY09A{p^qO z7a_*@^GL*Dl5>3o@)jYtgm3IrdEgviI5%Pq?BBriBj@;eTb zyy!5=XB{Sa$zhVuISe_Pl&=5!dR0X~BW-SR811v!V|{LK&2b@5IE?i{bFaf>FX=GZ zyWueDXB;N|DThfv?=b0e`2_wJ969N8{Noo43oB6^GM{R`EY$NUz(%{;9dzvF?e|5kp7<$tLf zZ*MjKQ~~wRdX_JZ7&(4o-1sKOucfd-@`kml@eV3;X^eS~YW&`l;JCx6|D`_jMb-Fy zir*`Eh#CI9h5A^UaTx27rJNs||E27|&HqyN-;TdaXT$a}#2J4%{blU0wYQA@v-Xzt z7=3GRnf14%8h>1`)UuS3HzNOZ9eDEn<-N>*tLi@y0VkP%gslL|b9qY0Ck;2%tC}yOJubiD za3y$}`7u@VZM5&@v&>sn?K-sQ<#Wu>soE9b6-blXgSLnGD`L#YRqZ7D`wA}KK+gU* zT~JkTBkl_JUo*;U#Sq8;u&VtW^0&hJ`#Dwn8RD#w+wxm+i8pQzdt z%5!DT@RAx;|2g!ZmF)kLjjH~2G}cEWEZ?K*r-@_CCslnG^Z%o9<_oI+>#+Z5w~+fH zewz88s`^_|=-+1kj;jAQamnx_Evi0``dr27Kk~Gy{}YtYDo#J(gweO&MKSF#4TwwW}s(+K@a}FbK^si-6du!gQ>hD0_ z!ui*H9L)J|{&iLVJ(9COfqcruxBj=}nA!gp_Q%%m>K0D#O;vjo<+Iu|Y}@l{E}!P_ ztJ(v|-)fuwJF4~s>Gv4@Xp5@ttp+EUH>##}XpgJezvy<=iZafR^K-ArB$ja_9VFAF!H;4F3fA#Uz^`G-9~Q5yEW{u&F>oauW75QnT7vr*gtS> zgg9g3+xELA%jq3g^>b0k?=Yjj-$r??DLIVui8auev_6GCcsD>uujKk=EYo{E>e7TnMvuv+wh!E!-`DXLa z6H)uK{;X>;a$BD3Y*7Yf?1QQrXDn(Rm$z*{>k=#7wb|i z|AJ~*u?w7b80EE&{a^My)zE_Wv@UPtfq!?HxxCgDnXP}bZ0~PXgNMz>x{~4LOH{)K zj6aWY{+91j4ZD8{%=uaVhH7X>c|OMZS^kHr;Q(|W>*4ss82=yR{4M`0)zFFY>#<&z z|GjGH-UR0SE+@w1_!#F`>qo(ptWWWYIsbwD2FuC6Es$p%UJ0IdnC#y&yy7v{5F`J% zJXXA_8ho^e$4V@}q8fU@?~5DO`w_%{ALq~RPu`aha?W2MPYQb}q0jjX^f`Y)dYr#N zo)Ph<97cZLmvxx@<@^Wnb3)GL5y4nJmN6PqYfkf<1G%8+;f=ZF^5SWcbMee4wKw>nB+YUlRV)t$$Jgk z{lViYhtYo@pERt?3+?Ig8$v$qF!m3R=N!iR?eSXJdXFAc4U_O^eZj~zW4>KqWKOAuG{(pEC1%Vobe^+*j`=^Th8t)f zn6h>IT~(q%>mcn|oNqfc_qubn@hg{s#6Zus>z=1N(VqUXO3M!_4#92J0`+ zXB$dJ-cV8vdDP#=h+*u%8vX?3v621JJOJ)7@|Kf03MF~U;pf2YZ_971hQf!y>|e`| zRl|2s{u|jp%~*eJESUJK52?rva@)@+BjqI;IuiQ9m^lQ`RymBM^ujPmF zzer;LHKTqud8|+8^YC{Q`)|)LH?jY=d^g=N`nG&GWth3VHgSGdUssVSjOUxCjeK=h zMY8aJ(=CV5zBk$Y+)|O(p9aseoEW}snlrox|97J4i1{bHXgpu@jEZ~?y8T3;Uve1jDaQV-ZB-3) zzS0^C%Ucs{kK2Fi5Hpugt4*KFr*%r`=Y;+(W~(pCtCjt+{49l^<%T5HvA%9s4&!-hJNu{gw}7LL9Oc#SF`rS90`@!Y?C)dOROH9-r@dFmlgz)aB6MEY zJ|yI6=GRoDNcN|MJj?taRMiR0ckQTRFe$c>LHp#d6D8N1xKaD^|z<$|M6>CZY$>CZWg z^tVMFM*7O{ecL_7|PKtly#P zV zzZ>PhBO&A|!Bd8d&CQyV$=#e%r`# z{#rc+f4kUU&6W7)rD22mleb>g(D~*w@R2x>_b~rJRo@~x``gIrJ(FblW2%~-pFP9= zdfQZW2K)VI*gu@3)KsJWJY)U&0bV5}dj(Gaf~sk%0hgF3@#-P;QC!BqPpcXa`c&uS zpH07;{qRxV1bff2zw39X>Lm1^Wq;PcgjYpTKhJV`tv`!bACaGD z*`M`SRLzJ7W`8u3{Dw&n=PR`(oTog?>EnE)_Q%9??9cD2+BwwkPWVLSgY%Tyw^05& zVN)~f@3iUtp{lEHhCIvh|54RNh^Gxdf&ZIm6t;He4BPeE&b-6uZ#!>0Oz9UKCOP{X z#4if@tiu$)#FzdeXU^=ap~J;L8TLqcEpyNCUKvRT!| zP+xm8Mjy{x>)KHsdu};=23#=wv29_Bw&b=@Sl z>61Rmx&CZ^_j3BSy!Uc>+5GOkZR{;AsJbNj|6VS?4fwx7rr^uoStH-jsp^JsUcI-( z{41*N4ED1)!qer4^XfV}&v}mh-|%5oHwk;sd8|*2_Vyg-N6Ux7oFC0`@Q|aw8Jsa} z^Yh#^GuPj9Ic9D@&vAY?`~zMM#`x&7e>e}Tn_de}hVA*2VS9c$Y|pNB@q)XrF$`VXPngIfq-!Ke?~lC)?Wxn*qb#KIjAtd;2_xVQ*i| zVY25t3|sqp9433c4wJodj_HH{&_U<@L_C)zTKj+9{Z@+RF_V(i&lo#yn zhfct-x8HLZ{_T%BO!j<-$zG4cWUtp@vM0)K|D+=)dufNs-VKMz-ju^+FY7S*H|;Rl z%R5Z=ZaYl&?l?^LMEUKXbL6mhKsgM12k;Hb3-%5`Ct%n+;5iKc4#XTLd%nYDug77s z*XuCZ6Xkbc(vg$Bw8LcYhQnlU%3-pXb(s8{HoS3zs=I~F^nqIrN5KW=S5@7&Yr%Jz zhg4l*8Mx>$#^(dG4wHQ@-;Jcd9X#jAJ^U|+;4_uq#;mHN^YMd7Q#0uk$C$sN>b}O8kSHgPi;H+VNUW@b9L&{;qKh$E_p06H?IdamE zJ52gLf)gS>msgP9kkC&HzTq&XmvNZV%Nh>+n-=n%kl!}EDWmF&Sg#%`2zk-4t)D}) z4x{`Il?2Zj-t=!&-S<)chm|9Te}|(EBmQA7Piy~hOvvMoJ@|LHN634bc|CtP=`hyo zhld=-`sr|Lo@qUQIPEaWZ!q(E{&2?dCaf1{Dg7yCZZC(k4nGa~w8MBFdN|MgLsj=< zieGRT?X|}@{1n!|b11JK&d*clRV2CtZ2jl+_a5snG0LyU`fKIvuP#rLv%j`{d)VJV z&i)4Sobe}+-*OoB*TeY<FF(1(R)C+lsmxGIjW51#5GVtYvlEawaUYKM4Q&pFa zD)pjb{*S8ebDO{shtd9DgbvL&F_NQyyyzKjtyguQK>fYQ`Pb>;{NzQXN&2mxs{3Lk zIAPf4|HUC@-ru~KV&?LCaZ2dtg#Im|f7>w5QzLYq`Xc8a=cy4oPdyqja?Jx^>wlYy z(0S_79wToh#``Qslgz^^(oXUl4x_OjW&c{gr6P2GdX&>|C4G{YjK0n9OPs!~@0X%1 z=ls4D7y9f^pq~)>lR`hk%;l->`9Xuie|LSpiG_i<|56cJJ}R_EXjg+*<38#>^zlGn zb@cJyby`e1Os%ejwo(jT%v;@l^u4}}%DZ_`cEO?+DLecHK?^3fj;e|w$# zzP^itS%e!M`|W(6?Ei|RPkQw6@clj9M_q?L=nMKoepA_17*qv>+E>FBDZywzx=dNW zg7nEYbEOxSS0;o!E94@6l^2#*iTG6_ewBz{E#g;;_|+nQwTNFM;@61ywL>aYKDAUH zRHvN(TCzuF%3K%2wNsvTS;2E*j%2_X2Wa=nx8NMuiQm)`j^C6F>o;Y=7-wyIO+{Qg_Lt0po%YcTr04!tKY9b#j{eeHVW!_J zxpGVqAzqOU^U9tuuS$fuWh%^TyfCj7`g#~}{9BiXzLVc|`EdOA#lcQ~>HSzYzfTs! z`Wq$%=fk{-^6%6iz2D}R2faTQ;$oO%Enw>F+#X{heyhkYy+7sJYb}PE-XC(?H@!dP zw*RLiV5fYZruuNwr}uqA^69X=Jqvc)$Ce`4DSvt&!A+muKXB8h_X%A4TeCv{c33}7 z_8foX-C_NBQs`&Fu6_YbHn@E0dA#cnJzsbAw+)5$>G^p`pX@pIJAkfzdfx5o)AQ?) zKGmP&4?Uj_>63qs{p~$qr+n#ou*;p(VctRZ9esL!>GJMGnCX1qmDBmX%X_n7rt@T1 z?k5G4KQyOvdHZ*8?ew>Ov9Nq!uaJxQ`$YWbMSMChbnQPc;?sGZE2r}`*FQQh3&}-% zIn2+j)53C;^H2rddPg;_Uqr~a#=LhcEU3-$#k1SbWj1g8aO1ZM^31m^`81Q!LD z!mPWolmF_dkb8pTf_=dW!AZd>!D+!6!CApM!Fj<2!9~HPFzfBElmD8ikb8pTf_=dW z!AZd>!D+!6!CApM!Fj<2!9~HPFw<1vj<>Z@!Jgo_U|(=Ta8htea9VIia8__ma9(gh za8Yn6%=&o2ssFmDkb8pTf_=dW!AZd>!D+!6!CApM!Fj<2!9~HPFw^w!)_)`_*b^KV z>LGE($J%nU+>={_CTHJ;8CozTkx5q~Mg`wBU^3tl*sB zyx@Z1qTo`PX({REzac8v6C4-p3r+}53Qh@53(g463eE}63oZyQ3ND42mfCLq8>50f z!EwR9;Dq3$;FRFB;Edp`;GE#R;DX?y;8K|B=)lc?Q&g}gI4;;1oDiH8oD!TCoDrNA zoD-ZETo7CoTnaO7z1;jSi3;`v#|8U>6M~b1Q-afiGlH{%bAt1N3xbP+OJSz1otyth zqJllaalyXegy5v$l;E`BjNq)`oZ!6Rg5aXyQkeDmsx$vLM}^!I92e{hP6$p4P6P?bn8D#H=Oc_dV=GEeZdLANx><>X~7x6S;0BMdBFw2MZu*o z<4I2_ze}TnJ;8CozTkx5q~Mg`wBU^3tl*sByx@Z1qTo`P@gy*m|7B6Zp5VA(UvNTj zQgBLeT5v{iR&Y*mUT{HhQE(~D`uT{{-j_#(+!GuZ>LG zE($J%nU4G1{=OnA*b^KV>LGE($J%8Lw=G^1m`F*b^KV z>LGE($J%8Lvc!^8aX5uqQY!*cY4-oD`fAoEDrBoE4lC zoEKaWTohaiGhQJI<$qOFuqQY!*cY4-oD`fAoEDrBoE4lCoEKaWTohaia|_Vz|MZ)f z5PO2-f_=dW!AZd>!D+!6!CApM!Fj<2!9~HPFs}x>`CT0q><>X~7x6S;0BMdBFw2MZu*o z!D+!6!CApM!Fj<2!9~HPFh2=&^Z#U2uqQY!*cY4-oD`fAoEDrBoE4lC zoEKaWTohai^9G=s{|!;Wp5VA(UvNTjQgBLeT5v{iR&Y*mUT{HhQE(~D8-Z^AH%0|} zg5!dH!3n`h!70IM!5P6>!8yTs!3DuZ!KE;70=oI%6cy|Vjtlk$Cj=)2rv#@3X9Q;j z=LF{k7X%jtm%{uM(9Qo-QNf;vTU4+oI4;;1oDiH8oD!TCoDrNAoD-ZETo7CoTnh8kKsWzS zM+JL=Il+0s1;ItZr7+X$y>9;7qk=ubalyXegy5v$l;E`B zjNq)`oZ!6Rg5aXyQkXXb-TZHk3ibrY1^a>%f|G(%g42RCg0q5ig7bn4f{TJnVcr6C z^H2YNXNWz)alyXegy5v$l;E`BjNq)`oZ!6Rg5aXyQkb^_-TZEi3ibrY1^a>%f|G(% zg42RCg0q5ig7bn4f{TJnVU7dc{Kun$J;4;gIS+{Yf)j$1f>VOif-{1%f^&lNf(wF+ zf=glE26Xd7{~lx@OFSsDMD7X~nZlIh0 z?xVOif-{1%f^&lNf(wF+f=glE4RrIrJ1W={92e{hP6$p4 zP6ZM#r^g-`TPLbK@#C zIy%yKp@00mxp|>~?7Z4CGIHfYU;mjiW8=f;)RnHj(X)t1e@BJ~1}}~enpmg%aYZRj z45%&R6Jrzorx9Mepu@4zGyN0&fg|eZk^Kjcy>d`dQe7%FHh8v6UF@Ie(wkM}=e>SD z3IqJ_TL@DX&HtY7+>#Q9oqs{Kdj?x3ama|(@R&DW&9*5;4@`{;IH+cZu}0uHh!RBE{@Hau;UMe z!1e#PAX}vT{<4ST3{s|km0khWWwODn?C*Ma*_1E%jxXPLx{GRSx7wDR!2Oo0zwdYUdG7Pt=X|G1KeqMhCl)SP;NrEw{b$#r=;~bd zmX>SUJEz?;_{`nZ;i+#?%P*gD8v^fMzr;VhvM})Iir_zY<3Q7$F@GAL+wl1X?~3p@ z_4udWg||9}cm*Fuv2)=+zVVqqckaSjFLv!nx9^QN-nejUV5YMSkFP9tjwvwSetde} zJMkCpNUR6?A@Zu(5h%SY}G9QFD9hBRcAQk__liIC^11m42~3|^AHqvM=1)KP-@WDcz5nSO zkx`~|c0&k1C;cGbz2bN0NgoY^;m>Tm#`J^#-CMpY>9>XOXYZdU{UG1H;@9R$KlF#` zGn=k4{osH1mM=>BY`rZ1d*?|%$ak;!#d*>X{crlr&^4wX{O{iKSxLWB(tlu{^n-l& ziqFiGepd*8X7e?sAN=p$@=-~@Thf2uJn0Ad?iEMoNq?!N|KK&IAN=p$GAZfbAn89e zPx?W=d&Rza(&y;J_V@5LrXT$8-tu8df4QXp$UNx>`R)}T^m2}OyiW9A`DfnAY0tj9 zx{1?wcz>_KokQ7){*I8&m)~W=H+=XLAL;S#JKG;wyhh?<3psu!gj>5Jv_CyQb6mZU zgC|@ZpEvQ2ZpCn|@vq%?wAT2%%crBq=Us0)K5zH_+uR5I-HtoCo0z(DUq~tIfSPnS7jd>=F3vDip zkJIO#^zpCkSnO_OyoeN0yQxp&4=$$r7*g7aba}n~bl>gMX>)h``@~ls^Y_;fE!)M) z&Q3>Fz5V!am-ufB@qhV=5I>p=gJ`+n=q^S5R4Ocx;F7z^B)(@&Q}zu;}B~?U3!+wl9Wf zz4W?&_Wh*I{X^(4gQ367D&L*xXM>@??Cdz;>vgpMQ$C&C>8JfY+9%8hul@ZmdH0Fa zkNf-n{*(T`b0Kp*+a9W_^0K`QK+gV#N7dhk7|hovd+ccc_4c>_P0IP_{?2@&O5R3$ z{~HEFpZP0)zji%uWiZt)`n*1_r(JlvG3TS1WB1H$7p?yF_Vo38qOVtszOGmA-w=IO z{e9weHMD~Tcl&~{9@u0G{SEJa{B&r4y$ixTZuP6j>Bd#sHQNm@wr{qdRzLig)^|6H z@qM(S551FNvG z`5oTpkC=7=AYEAn8c6`oobctfzm3Y+isJV!4ZeKFkgC?0)YCS5Mhc1NbaL3Pu_IlwHLH<07!C9O06Z5(Drr^lsPFTFpL^!~)(S^w0>{{E`huWm=y9%4^s`O%~t=XqY#19P;~x5nA_ zl(rMt4)y+$4d3e?Vszify!E?MuCMv~g>ybS*N#q-LVK7v{rA3nC;G#@&T{kG-yhtM zpAPf>g`0vq`!D;~iT=w$FS6c=`z>ROmihg#-bHC2pm)&%-+tTNDyk>gN$ZtcZjVELpwX^>pLgoV(t*N zj&Tv%N^svU?efH67*}(jpcL_ZGWg5=c!)3esgRE7XY=R_FZ+0pe<_U5&!gXR{&)@d z1mQ;Jx;KmqMq2#d9j@i@Q!qNeHx!T@VRiBgt}Sa z9PMTu&t=Ag@GJ?#^B(8(W_VUAuGnt&_y3L0&nxik_qTIF&Umgq&VRq-=W1ymEB$@{ zOW*If874ffhwB+PcY3R0cE6z>ueIOo^t}Y(mEO3*jq}}w8PTVzM?c=U&!Gn44}afY zKXanT*I#EmD;NIQ0@_DRml03hO;_Yg+T-(R4|OcrcP?J*-8l}sUjr|?*E7dac)>Vz z#OLcM{-(fe2W*o22SY!64eqojR^r{ww-JE(=bi15Ge6}C?s5EWbMFSRy><9@wsXO| z7~QOP{IqJf$Ipg-#`c@QpKbq4kLlB$`Rnai5ca3QbSKj?r$648i#-(UC+>$d>xYHL z#r_fMrOW4!;on}p{E4rRHup#V&Uko5eURzY=4Z8#&pC2<*e~NF{p;~C#&@Fsyw~T6 z{)*hs$vuomC;G$qbD}?tH*tJCemdy)dBhv-;CqmPtJ%R9O+OffKh_J!A?hpG!L9Lt zs=WSr?aQLCv;H3Kh!bK@O`pE#%fxZe!p@-n82w=R`m7f_va+PKke@_UN5FzSpKyerTzL=Pz9UxohX+wYsZPq54DL+=xC|8c(Xz3!NlW5(YnPXDP7ANzN- zW1dGoxGx&}%ZbflKmWvF*q871R#+$p%XMNf$d8}79O_5r+vzhk&xOIC_4nyBf8_6T z_X~c~yB|fm)MsnI;G}n-IP=H;9`_%PpAG3Pm-RmN#(j<`-#_?S@1J^RKRSLkv=`cY zx}6jTf5!XEEfPBe<6{E zO!$S(U-q0kdnveY4dp7la85Y%_9b5UcfvtWg%`dmEc+5K{8z%V zFY&@}1b&>o7x{eZH$C@-YNcibY(DSu~iS?Lg*p`40TSbrjCD{|hqTYI~%D{2zZ=`paU* z27iNL+-AF6<2kgyi8J5y`k~!QpQD3c_Z<5p`vo-}#})3gF-P(J1Lt+W?#G?s_iz4M z2!9rUs~zs=k)Pa|dT?JE+>fF>%zy6X*Sve-xzKJdD7n|ebD@1{cdysy5CN_WTm8+& z_zKSk#qfm2B@+3T+deVp;m ze!hMk`J8{ged!$i)31EHAzU-x&GFYJqj^8mU&Kgjycd2hbmuw2wP zi=^8Phw7c~Gv1>Pe|_t_kNfbgc6VS-d~9DjKXIQV4WBkYy)J}ra|IvI^y#8+2XS9) z=Yre3ozZ$K zxG$U|;EL^*2f7J^Y*EZs)-q4-w(&3)K?$e0> zqq2U~?Vjz(#((^5X!o2)P((j(BMQtPAHiSb_ng1^_Ze3We$kJ}d9jkqDXN zEA_seMhY(4FI#ahY)<=daX)SU=5X#Qu9x?3e%#AP`=9oAhU2w=^HZLu&)(_ph0Vj^ zJ{aEbA3qz$HLfGP9egf~yM>pY^!`q44)Qo|@8A5Gcb~H^ek`QNanrB&_ zr`#W7Z3a8t`zr|tvyyR9KUC@VXFa;O14p0A`11B*xw0AZiI+n?#P`)v|1A#l zTI7kdLEn+%{L6Y~?~Cq7p%&(AIV*wpd%``myH-CP=?(5jkzem$@_FI0;65V#jpb*k zqfndoC-Gf)RKmyfgue_%^L%4}e`qh-equeu_Vj5y--%DJ`#B#G$H6cz7rma|s$Vla zf*}02e0ZF5_jlXB;`nmmlV0rYafZwNeBgM#a=F+cY@Z24qnocSAM=eby>7;rW6ro1 zitBWK*dpzDjQD+pf4Tf+eg1M+$q!rpzkYtU&6^)I!T$pJd2rtRu#5le>t}G@{BY;x zUq3%#{)z2}=KiGuHEZ};|J^I`t9~Fe2@8SbHC{CJKKBwzFQo( zSRSrB=!aJvzi0>W;yA=9wpX3cdwb(pn4sF+Hb(31LXMaF<2l%~-r4WdaepP|kM{4I zuag-I<|7q8ZNhU$mgV7;F?U@X470aCKEIdAL%&=lw)T0RM}2V|-nn2ohnrdX<9om3 zr^RkV`4Y&PKRPp3T3<&-viqU)+D7f0n-=pI(3ZC*+S= zzy9_g3O?_;4a%efE@bfsSDSmM?Tm(3Y(U6|FISL zgR9X0*S`F1E*MgLFTM-zY)Agicu-;S{W;gmd_VOf?g{)Il{?!9+sWFS;DC$a$x&Z% zpAhGxyqx)B!o3T_dzzgK|Bcrh!!3fW`TjU`Yd87@?n+P(clz-3$1CdL75Jl5o4eh+ zvmYXw=6R;Iw**6z`J{XAB46IN-bL(XZZX}NR4+~;uwQati2aD}LH}VrOy*OxYq^f# zxXbq#3EX3QVfyU1Yd4|;b8X z_7(Jn_Sr9XleF8MoCjvRNFZ;WCkp%V$6pTXhTNkJ4ZHI(6oB_FEIsazhxCr0!@>a9 z^KfSUUSJ2u)jCfU_T^=K#`&X9o(^@)h10?qOZo;;`;)v8->J{=@pAB@gS-76*H= z#bJHb;%8^2AM6rpHe9itMf>`B6oyi^Gq#)MQos8Bns*P~x%s`V{e!a3IUVAqUaxxo z3iZkwkL{lNCCBq)tZwRe?t6>B@p*2}Q_^o@Ju%#!`1HEf_>1_F{@a_ULcO)QhkbgZ zAF2EMiH}^8drj^azGUEY723+D^J|MO>kzFb-#X`_!fv%7ZsR*3d+)<@&*K70P` zF6vF%k+s|6xnZ8)$JK|C+xaMeGUE9qkYOi~WZQ#`hXzUMKp)dz9l}33^=Fxn$OR zjj5jt{y46s{mkoGz0Y9l3iSss?iI1)uD^f%y3f&k_F>B|{9pbv)3f_8t?&7leEFN> zEyh_o*!5!T`~9nYx~=c`|6B+!1yAn?6eqe`h{-jp+yfySLmb>4yn% z`s~lolYWrzUePm8`fNo^|MSXauX!^{lYfL}*-@WCpB>iBhPM`hddD0K^ z-7D(zq#x|y=`&xr#`J^#-CO>Xq#w>VO`rXhdD0K^-79MIq#x|N=`&xv#`J^#-CO=p z(hujSrqBMHdD0K^-7Bi|q#wqC=`+82jp+yfySMzVq|e=Nw4by8cAoTueD{iq_owIE zGR9lm9yrdjVfVUMy?>5(Tk%xr4|c`$>61Pk+TZL6(`SCt^HFq5^7FVe#qBHb2Uns0 zkG#Ac&5$dRIDbsY`OxLl-kTrinJ(=w-g`S2gz+?V_gV973Hd($r7-?3|Dg~*`oHk` zz{kII*{6Hrg{r?#e<_?_EW8lxq{36-9P0j${IQoW|Ad5lxx@3wV}a+KciR!%pGRR? z&I@7t#4Ue+aGyRZ_Z=(+^b!5h2*b5}INYZ{74BRgut#m52|V%TkiV#xX(|*KzZd4y zi}4%_{-=LJ?)H4!bHSb8FYCeMblBeU-|wX5v%L`H%O4H*?Hj{=^r=^U`R%&S+TFP@ ztj;>8PWwE@7tgOBMfsS1bRXRu?A5Rzs_gw+Li!_;4?lk(J{vx+3!3=}+cxgYoUd^0 z-~1na{&y~H_w8u5nuFOk!$#m^Q0@zddboPMuLJ$av`)@{Pa&-Qr#`#1lAzfYW%b)3le56b%SOeVy;0-0jI&ipH{ z=W-$4SdY$T z^L)VF2+w^FmtfrB;DKxUD-|Dq?sPck%6Qp6qg}ke|C?UUai9A{%m?LhJ<;ZV+1uZv z@_kvY2ll7==cQpLkNvL4{hjj@Eu!nUn|FymcQosTMcm*2{hoUZ3ew|28s*XJJI>&2 zXB_h8t_!Ip)Yn;m^Xp2ECp!FH5XJFjZg`dgayC|6IvmHN*iSBe%8O^EANJFOy+*yh zz4pFV^cub|Hu{n4ZRf$h(&ulO(;xJ{c|3o^@dHKm{heW4QpS(h}?Z?pTA zE#0ZMHhgeJe}S;=EGi}YQ5Guqwqj_h-kbkbh&_`HA*R}@uzviRUyMvFa$!{frxBPp7l zf9*%M9q0H_F)d_&--i#v`EL_U;aQG(FdQRqV_Jvtkzv+-9M7~Hc%8)OUH-|r&xLyj z+w3oeXaABz>D24`R>)XCjQ2~yUkCT~zADdrYw)%3>}LR5R@Mo_Gv0q6{B_tg-YI_Q zPp*Y@;{~^7UC@5z#k#l=pEN$K3z~36d{`H(_bNUY@xiq$!Jnw-Yl(~Di})}u##_bb zPw~OE>@NI?ap7^{O1g$?R=Y$x{(|FT*27kO?g>tQRmA5sK7Wo6pNsfh#s}B3RrsUR zSnZwtpIK>yQP#UJe9t_7s64qAuEoRO;sd9}Z1?{!K2gVf;m79$A5OZgvot>VN`&9T z&EQ!s{5od17*B#nmTe$7`Fdg3PvgUSVLvP4LtU=Jr;1P5^>EAX#j`Fa`yqeo5jtK> zf4Ac~wv#*MIf46tJg4!zR-UtXexE!S@%*?vm+|~V@?6F93-VmY^O!s{69?rv!Tsa% zoW}h#@|?x}XYtv9a7Ent4PNFgS%|(P&uKiLk>@O)zbwy1JXhqojOX+6T*WiLGs1Z4 zc>ZH~W+#3X&s@eNc>W7{F5~&H#ZLm|e^;KmCgp$2vOk^5uCL56m;i?&T8D&TJhSf!pB?>)qjlg^3prj_pl?x5Jw@wwxxX4!)T1kDEE3 zA9DwC$415u&K#N;E4ck*$K8?K8;ju6|b=2)U zlzVZbEBKu34f)yY^0~c^(h*7vh_A4JWY6J!``qN%L6_e@Ga{uswBJo19y>fXav+z_ zPoUKKLsK(Tqf?Xa$Yk!|NdC}R0mTn6GddAn;Rik^rVdTa97pkAgtx-T#J-7xd&iEs zc%K;^2@a!s+<2}q?%*&wnJW|~_PV{fnVkOv?e294bNPu8xbDjxn3z274vig{ni(6J z$ZvF$Q=_>_|I~Ec)O6ewedejN;MG4e_K^c)NMZ(6bs+DKP8=E=VbNbKOpO%Apb`H- zbp{_nALB>G|30b?8ZWrQe*D{m9L}JX%#6A{`F;0#SU2iAC-x-+kgk3BK<&RI()XJ~$j2n>l>wV1h_S4vp=1w0d>$Oc|0MnVib)9ochyW~{(O)+F>f z{PfmsYm!wH`AusQHm9k519)D|plFGkHYS4ylMp*v1Y(%le>gufHaT`+V3lw62+KQi z=1YI@dXiKr`UE?X)_WdG?xSX?%7NV4hYO(>eQ08LvMM={E9~Ed$JNP-hZ3~KeI9&H z@7t81J|(LMXQsvrEXTTh?zk^CWXvMF9Ri}c!YAbi$Bxa6?>!{B7s*4(lbOu3+jndq zdE(he-~Y_kZ978sF;7h7!KkTykH*u)IYy^)jIXeaW7%R=k z)hs3ivG`IP`eA1BV~6&Qj2*!UH!^j2CV$v>Mz(Nt|JW6`?0oiFw1|-{nVrbbfbS%$ zeXDpd*{~j-@!YIkszM6AVZAg^KL|iDt4dU!pn4fCW)2-54T{*9U^q!mtn;Hfo}e%P zxF!khBslpd1E-0-fhYEU42c8{P)i|N&u=v(=q9@wfYD*q==j*^{$|lUg>L(?WNfl9 zmP9Si9VnU;4`SJmnmd79`E0BRu>^DKW*%h%gyP(w5iqIjEk#*x!7zrm1Ot9I( z+sw@5npP_LBbzXwaBzW?1zqPSkI6lgWVZB~sREjG>e#;3$%79j8$X7?oy}?tne$F+ z?2^oe0iKO2c#s)avVmwHemL1k(d>bsLMC0 ztyX8^XfFzkJrK6v!57D1Sz%HN({e1_tf_oWl8W<09L-`3xxGJnxG*zvXlm*}h(P9X zBqsgf@uxB{{YFRnmeC9HqX;4Kc6N$? zaMOJ8abr$RafXes9xa>AE1x_FuN>D}F~y3X9LW2w*ot!_@;htXo};75jZpP$yFzrM z67V)>E7lE+RZLLk8qIj^!(tJkAhD&hXMb=L{dTGe2wuY!Z1tr5Zt9vGrpq&1hW>`O(V}dxozZ`9Xq|% zakcJwB!tmjhgN3G#(I+$s9JBFch@9CvsyLu8>ePkl(A8BHMVaO%XgF_EDJNwZQc3! z_L1SopQR}}XG*3K|A=^f=Fw+AF!Ic!+ur~5*2hOu$>tENlweLsE(;z>&{8)izT-#J z?y6NXxe?lrlF%?iUf4)tjrfeKX?^-Imh_tvjshoWr80%tcEej(4Xp++wQgecKpu&$ z!c)H4s0cJ&V_#o4xzXDd!8_7L@0Wz>wYiGoicWN?7DR7rp@arBYsKl)@yf=Iilpr) zf(^}C0Skd4wmYerP47$Yy*J6oCd}q!8`i7I@K|o|*rBF{)G}-fheoA^&#}RyS?jal z2X3}uYJCMlg^bQ!mwDgOnDPOzGMn%QNL+EvyEH@e2dUTj3?t5Vp(FZK;>s~u8*41u z?Ri)ftjK|o@FP)va2?m*Y{|?S(n2dh@iCahq_v?Q5}BdeL$+GRv!ox)=O~X>`-f5- z`v;nSl;sYuzIt0z=A4eixf4BU_Ed^Idyg@xR(N@eg1FopL>0j;n(jXTVUkUJzQWQZ zW@iu`5;^u+!yHL9`Brpj;>GcqB)4y7>`(%`51Bf$DH-=|X-#tY;3W3!lAK?6AHWJ& zBrseKj1|HvJG2*H3&9Fv)Nu>NYd&tK1QoZ+70a)UW*>N7Yu=ZO_B=|3*pNMKzs=IMS{kLZi5E{R2!8x_|M|6b)4wx|$uqFD+MJ zdk>L-)ZC`fQxHE#S?sbg38qGyZ@w-{s|J5GG&f{Kim= zlNPt;uES2#J9=}LR@Bj;J@|(O0l`X;s$h_r~hp_}$@Au#QYJeMf*SHXZ#+yxb zu06K-t=CemiLq&rH6uKznixBbH}+cVwB|VcmJ7BwPlT2Y8kdZjSmMW_giX#XC~VZ) zl?Vf2aGN_@sX@Z)JZ=qQ3lSS58{&>&=T%3N=n0d*-*UXq@6Oz}VYMGSAAcelogB*@ z+U!Tp{KVKO2E_xpW-!T_#-fj2Q44mr~I(o@+Pol@nS9Yy`>XkudfTVuut%)Zh_uM#$N_ zN~?mEfka}PjyB@kHRw_4SoU!d+1lUQu%5%|APV}x*oODM_klfJ$kNNec=98&{oRMv zJUtHGKelHyw`tG39@mS<(e-=Q=f*bNKd(nD9(;`J-@A9?*q-(K;8E;0CXO7S+1+=) zZ;PVn9C8+CFD&!pUOufGcz;ha!5#nBhQALQ$1&E;tewwj)8%~qgkfph_N}{~d=A^= z(uYuGMaHWT#uy2dsWX3!(dkl*?YFJ9u?Lcaxy z)eF}=+T@a*%;wY=_Ee$06O1PsM*i-1{ICfNA!BJ8eZf7vkWgwb_;p5eU7%A4yNBH6 zg#T6>`_Sx-P}K5Uj85E!!W4TnY?p>jsR7=+@rVxT%~?9If%pkR+Rew|$oTW?{LZ*9 z5+*NfyhFfE+t>?PKi4|3h4DM&+S~3hQeyZyGu+w%7$B=>TOPrHn0Ma0rd-`QMbA6;axU*x;zez#AVUv^FjmZfvCl4RQ zi{_~p4^9-Y&9<$zqTJkEXeK-1B?qHv&*T`QSH_FpGmNj-uqs^hQsNix{ryq$G#DO%WzU`o3DH4Sm*YBzC%Mu|{LFm+{H(mu!6V{gUR& znJbwe`1T3kU9di`xwl_ycUOBhutj~j(Uy?XH4PWDY)l0Hfo?2voSw6<=$GGDwIT8L z`SB;%8iRQ{k>{MmN>}?FYON(cTQBVWqPMjN;M)xCxaKSLDPOCMx8>w_LEhvPLT;v| zB7~paNUpl-m0F&?LgqriZ?UPD41RGCN)dMC_xe}yEb)AXq~D{H!wu4jRv;9T^lIf% z2ApJo#3NsoVx!J)BgOq1zB7fv;2o{S&njG()76VKFk_z5!z9&H)uFdk5SSK&{Sld4L_zD<&F17`zYVMrUAo`$U7u z9gE{vjK5TNoRXd7T_hwYOezUTvgrp5Akqun)PXpWL!_+jCo8bJZ4} zcrwfR!)Q1iit}kWrHNw`ZXHHI{5>?~k2X$DjE)^Fj5%)sOpWgM4}SFWrwVaimEU(s zcyJXLzmLJs(>U0h#fKNq9PbL}l09$6U!IYr+eRL!#q}XP^Y?_hcP#b(Bi{^8l)MMb zGrE+22A>;tE%j%k$-jh8*N2z7jL823c)_uy{5ieE%Y(_w+*ly#&TL23~^yJ0pbyz2pPrD_w5ba9%iuKPeo;mxOrb)QpJJ~+#a@4kz5Wz?{VDeP zQ|$Gp*y~TR*PmjqKgC{uioO06d;MVyn593(UVnrb)QpJJ~+ z#a@4kz5Wz?{VDePQ|$Gp*y~TR*PmjqKgC{uioO06d;Q_d4zu*9*y~TR*PmjqKgC{u zioO06d;KZ)`cv%nr`YRHvDcqsuRq0%@4WC|pq<K@I)W=qEU$MgtD83y$teEn=aC3YXZjR5w&GA|IG~$03z6DfK{7&$B#kYW~ zirc|8#kYg&ir0XzC|(Wb7b{tw{|V`Jc67TQ#W#S5gk%0Q!ZH85gk%1*!henUZtCfF zlyHoH zNcg7^zJE=(ODnzsJS_a1kl&c?bC2wIOhL~aLhly%&YT1ARPUtg`@vr<3B6B z8|g2bM1Bnyg=77k6pr;%QgYVMImN7>s$$m9Wy5IVKL0Vl1Hv)CXfm_Px6Am?8~?|Q z|Dy2cP`=I`-R`8~4)7`AvygWo^0M&n(Eo=q9ti)p;3e5^*AR~SxFQ_&!5%Celz$%j z7iN5SBEEBq8Gl9jzZL$^3&;Fa4cCO3pQXr8558o`^8X$9=46o@P<$gerI_*|#gwNN zQ$DPi@{D52cPXYktC;d}#gyk2Q+`Y_Hm^q`oFB0{u_$v zzoRGUhvlLFKH+WPrC;xMDd8Ulcg=LVVa0vmtm50jMa7h#6OQ9YRXC0x7lq^aaY^{= zNbi;^>?Z_^G%Pe8u?Z4<)d^|8Mxe_hPpjP`n(R5{~f?3Ezu(Wyz};pM)O- zFTaHGN%%*=cfZ!{vWk1btBOKG;B^=Yw3)d08=cAqOr11B^-7tPi!u&Clj!$*EQ^L1^mzCPw zIU}zK4?@21i*2qh{4jXY@3*-G8(Pe72E6o+B`zhLCEv2drG;botZ*Lk8;>n<$Bg`> z@XtWr@jAv=;a>sY@cAXKthgIo5&j*>yFa_cT@sG^Y6wStT@jA@>c9`8F#o>&LMa2n z-#~orOR+y@xF{UsKP4RFFAK-`E5ffcf0IjGRk$1TWcTyaMn_lj8Y5Yh4^ZO&@bDeHn_z3v6a;G~c{4-$pRg5Q!yTK=o|B`TwuO>`8pa;fq zAAazK>HRJF_mRHv1DGe4Ry*A;;itjv-$43?%feA#=Y^xbs=`rU7lnTU@pXN()76E4 zoAK8=T?c-MfcYWs1t-Gqz_{H0D#lmEX>eBfLzG`a`4w*mmxRAc`D>l7YUG!NWB&PL zCrm%)ze70YKQH`U#JBWmv~OYl&`!sLOI%Sn0e3yV#GMj;0K9Am+PC7R;B&$|AYZ%- z?OiwrUUJ7$R}-EB_axXK?hpC<1Mpq62e4nPxE;L9a8Wpx z|CDene_1$|zas3%gUfBMrWurhiKKmk_>V@nUyQIF6s^g0(zC{w?q#{O4(xg#Q!x zeYYV0ir)d|5BD&?eHh1=^mVxb;h4X)aLnJZaLivu_-=&1Bk6Lx6yE^O3O@F0JJBKdhMkGm0tSrI_-pV#>!AQ=V5$ z`7ywSrUQtZ>dBv1hh2whZl47nu>cZ65a_FmJ#Fbk^jC7#slGD*gZ>j zp#KYh5Zv?OE>{!IlXs#23;#TL)n4>}#W#R2EA9q2g#R1(-;nKc$%>#K^3C7@#W#Xe z!i!)Rb&aDv3ip5)<`JLp`@nrO7(ea`@sS<4Lowxv;xu?zG3C32UqblqV_j}saVPkg z@K+(fq1fe0!b@Q%E%_A26XAYv@5wHAPVvp)itxjb-^k6wyF-3=l0T2~3r~V?{Vd8a z{Nv!RQz*ZY51a5Og}(^@%U{NPcu$C*cHFWG$}jv~@RIY0Pw{5(u;PcndBv2U5{~m# zML5n^=Y`{Zby0X9?6{>VtPg}gLB`9E%fdegUUtV~ms%O}8{;1qj`3%NWBkXA|B~^4 z%J@Gg?AM1?v~OX*KKusStKv~G{~ITczrT+BExCyLQTzxvCHyVOKU~9nBmCq$p#N{8 zy(n%6pA`NT$d{e$aHoWS3w*<8J6u^f?ysH`j{B<>CFlO?MaA4-tt;k!YR9`n{uv)R zCEV1na8tj+QNO!{Y3JU4s>5X!F9YX=zsvGq)3K=dW^h^f2H2hNdAY+?6er;G!sC$N ziI*8w#U0?9@Mj@!|9pq5tMJPZ{)+Hv$ZxM8e*?4I&oSXxzM^m}-$~)WL--}tY|#@7&z@pY^Y>AwSZ>aslMTj62wk{Qgm!q0-cj$ynP9-;gq@-H0IJ0={{ zJ1@-k)`9j`SIqWzS@`3O@6|5XHyH90+hOz-Q+zB36dV=uLS!+@x9=h;{4TIt8}dj0lqbUfHTV|1j2ssJQ}Ckibhuqco)zZ0VcEAk+_;e! zg}F|+qu$|4MqXC(WssjU@`~^p*wwu|P+rAd;JV?{?o)P{S!r$;!^k2pA0%sN91|C=Zt_t3t zE4~d}6h4FaI={ZqofPJ}ue$^LQNn);Uebg83&WR$30Z!5B`@vi1(Yq4`ZKaX}Zk~3H$eT_&@1VU2b3ccNDK9GS0$&pL`#;}C`!nJ3 zWluk!{5{fJj{l20FTCk4a7Vj4CcFdOfy0sp?vn5);lDj;cUKH2 z4~6o@@oq>sjz4MPINl8l{~qG&*@5;U{2#$9arm~Vco(>&cmP~cO#hdJWBLu@nEn;v zn11rG$**wqpB9e(hlM|Yeaxa%$lJhu!fn`xY#(ZO!$zJLj^#Zm9P?iij`=?&?B8GFFmpxt8N}a{Z+GW~xv%=p z@pe~LybOF%_zRHVvKRUm_WSWO?e4Ph??T>v4DCblZQ!1CC?Dk$;6!l>oL0ORJT4r| zR}_xrJ1HE?R}#Jt`=pD?$e*y^e>>Oih8~?=-x=W;-!9=8U)J!r;i7Q-zR@YgJilL7 z%=7yd#XP@%Q8CZ&*A(;n0GAXqzPe(@*HFy(t|(@F?y*qbn7=;7j4x5l_)>})->_oF zmr>08?NZG6vWgjBUNPf4rkL>+6*In5ioO06d;KZ)`cv%nr`YRHvDcqsuRq0Je~P{S zwix{>_WD!o^{3eDPqEjZVy{2NUVnrb)QpJJ~+#a@4kz5Wz? z{VDePQ|$Gp*z50cqd&!7e~P{S6np(C_WD!o^{3eDPqEjZVy{2NUVnrb)QpJJ~+#a@4kz5Wz?{cSb+Q|$Gp*y~TR*PmjqKgC{uioO06d;KZ)`cv%n zC!Af0t%UQKpB49j&k6rB#6maEEZTH+vM*-Wm|TYZdm(@t*_lQhXD5T-cva`3CG2;ZgX% z=@RS}#ogeO!i=v6;Y-5I&vNAFlyI~M%fdef|4SQ~pM{(I!-~28S~q-IvELt7O#dD4 z5Bg~C4-3cr+eA3(XFxIalTzUs|BzzFpE0~kG5u#1(|=w#re6|{`Z#6eW#MnGg1w3_ zOPw?F^C~>;>8fJp|Dy2!gYe7$675UL>A$X+{x2)0{EA}A-Ehz+ZlM{|@X` z#ogeZCqwx@0QvGO?QTH$%i#NaI^2-r9&lPXwx40e)K^9^^|?zi< ziYYHDru?L0%1eqVKc$%Rvammo;P5_J$?5;RV*0NNQyTgclEBeD1^hEyY9NV~XDg zt|+Gbl5n)o8^Y0^z9Jm$>Er{Ue9`}qaP*%Rj{b**k05`GPN97ZzYKooN$f8uUIsoU z?BDl)7VTT{&ESgge?a(D3GJmw$8H-IlI?glr6x59t> zr!XH0GrgM;zk52ApL`>@L-@n+zwnFbU&2v8Y2m1!Vd1Esyzzh1_%9j%W#O>jit`P^ zVZXJ*)fJE4gZHVz`;fmS19*S>Ovpd^A@Gp!mndI@^8t#V1m}etl&5gs!pJWQ$MV*N zV|gzN$MSY;o85jA;pl%rIQq{R|KrBLdy{LCTQo=dHMrcQ+|MIEUTJ)8bZva}*J8Q9 z1NR#p>Rqlx?!SgR{of&s?Pq^|6@TfzR5-e0o7cN{h*-)j>wEX!PFa>@{D52`8yW87|e%P?88gdMtMfD53ksVSM0-!BAVe9`|yf= zc*QU`XA_; z%{cn#FUt9dqh?KJ(4OuVGrnDh`Mq6UG5nA;crwEm6*GKQu@5Q@E9S3f89d=4Q#Aw2 zNr4%Z-;?7N{quWZnzJ{}W`3_q%lW+_&HSE^<}&xoamD!gy%5d(o`hz8??5xZ2cVhf z@*}gn1Mh6HZx8QkG0V$yz`U3~%gb|+ykdTMZc#JOC2GFBs>Rel&*AZk;dyRNGtZrA z=DD)StQwvZi_HA;T$g5^+tSQ)S(G+jeV!}gMW&qR zig-mi&)q~`iU*#Xh|K)*Tm&!rXZk$1z$?bba|)4t{uKNC;XQklGku;Dh)g-(^GBwf z@9iT~&iC$-Dd&6i$dvOvcx1}?UOO`7d@miDa=vGdOgY~ZN2Z+bg(I{4e2=S{?^S0p zCnQ|d2mSNCBd^GO51^U*@|qLudu!%Cw&wBlY#w@aHWw9h=;XdCFQ)H@#>YH*rkwkl zyrP`@j*)$M#XkI&+47WP%8QD9c*Q>a<0iafA6~H!uh@s*YQig~ock=iSRS@_?i27L z`}X{lN6(A#NV}UC~9Ule0*Io_0vI5xEPlD#Zt##Wa&Zn$CCEu3B69>a;@l;O1DXkSMCWsN*r!)e3OK5mxZ$n%DahD(OahAW1vhHHlFh8u=`Ls#YPVa63FhEs;qhH(^Q zR`{&pyy2qZlHs!9is7o^n&G15zrd%di2*uaN2OjaMo~Kudy096cvrUWVmd&Vz_F!X1H#+VK{C% z#QgNg{1`bgoHCp?oH3j=oHtxFTrylXTrpfVTr*rZ+%W8ocGN>GZ(RRIPSpI*`oh+f zzoI;CIAb_#IB&RUxMa9&xMH|!xMsL+xMA2YqILOcWGPMzs5GMqM?F`PA=H(WGaGF&!XF^Fzgq- zs=RSOCbsW{M!1rv45tle3}+4J4HpfU43`a83|9@;4A%`e40|JAm9K|Jzv9Gj%5d5+ zf3HG^&l=7fE*dTwE*q{Gt{Scxt{ZL`jvKGBzI(U`!xcF(oHCp?oH3j=oHtxFTrylX zTrpfVTr*rZ+%W7Hb*j8^f3R78*)NQoGMqM?F`PA=H(WGaGF&!XF^FzjUi zGnS9P!^SIeVmM_uZ8&2%YdCMXXt-p!Y`9{$YPe>&Zn$CC^~^4Bk75444zH*W{tlbw zwBd~5tl_-jqT!O^vf+y1s^Oa9y5WXlytJ8Bo*u)A;gsRD;f&#|;k@CZ;gaF9;fmp^ z;hN#P;f7(nq?=uS!-?UP;k4n5;rPBFwx7I_7Y&yTmkn18R}I$;*9|ueJ9&Q*^VefI zF`P1-Hk>h>HJmqGG+Z)VHe4}WHC!`XH{3An&Zn$CC$@|D={TogUrwpeJ zXAEZz=M5JPmkgH;R}5DT*9_MUHw-&@|Jp3SVg4?luHTg5wBd~5tl_-jqGA4ypiZxB zxMH|!xMsL+xM7&TL#Wg1F`O7q8BQC{7|t5b8!j3y87>>H7_J(w8Lk^{7^ zFzn>KNX%c4;lyytaN2OjaMp0%aM5tdaM^IhaMf_laNTgjFn_0UR{J;1-)YqHl;N~t z{$7aopEaB}Tr^xVTsB-WTs2%XTsPb>?BqO9v;GYyhEs;qhBJnKhO>tAhKq(vhRcR4hO35ahUCx-buj@mze$5C^}aMp0%aM5tdaM^IhaMf_l zaNTgju)Ak={`otTx;*@yNzEz4{M|_{<`c=J%zw|DxfN;j-b1;i}=9;kx05VYhO2 zetHZihU59r#R#?lpR|!@4Dt=#4vv+ zQ~OUF=I>@|dDd{=aM5tdaM^IhaMf_laNTgjuzUCH{Pq}549D}nu|CsAo-v#?oHtxF zTrylXTrpfVTr*rZ+%W70O!*BbhEs;qhBJoadFZIGypb0TmkgH;R}5DT*9_MUHw?Sg zru>Ez!~ES(Ua>rB!x_Wze0TJpH}az4lHs!9is7o^n&GOP@w|F0 zU&hF@hVzDthD(OahAW1vhHHlFh8u?6drbKaCx-buro3W#(}wvwrCOdfoHtxFTrylX zTrpfVTr*rZ+%Oyu^v3e^teKtv#Bj=R+Hl5j)^Of%(QwId*>J^h)o{&l-EhOOTRXeF zJ%$s*DZ^>Q8N*q_dBa7+CBtRI6~k4-HN$no4a07oDZk;waLRDnFn{-yS8PvN!+FC+ z!zIII!xh6-!!^To!wtjyol~9P9>a;@l;O1DjNz=|yy2qZlHs!9is7o^n&G>H7_J(w8Lk^{7Dl)#Bj=R z+Hl5j)^Of%(QwId*>J^h)o{&l-EhOOTW`v5IDWq>)?doV(}pvKvxf7Ai-t>v%Z4k4 ztA=Za>xLVK-3C*B!-?UP;rRWrsGp3HXAS2K7Y&yTmkn18R}I$;*9|ueyN#y&h7-g5 z9aLViylKN3!&$?5!$rd-!)3!2!&SpI!*#BwB@`~k445tjI4fA(ab@;5| zyy2qZlHs!9is7o^n&Gv%Z4k4tA=Za>xLVK z-Fr>>4JU?EhSP>KhO>tAhKq(vhRcR4hO35ahU?A|v!|2>8i!zsgQ z!x_U_!+FC+!zIII!xh6-!!^To!wtjyon2jj{9RqmiQ$ytv|;|Pt`46y%-_+~@}l9A z;j-b1;i}=9;kx05VfWDN{PY-(-_MWZTgu4OhBJn0U(FgbP9t(zG;HsjVE8k=1A*p4UG5ehT8nSxt)AeS$=BOCKm1@CiY z-=W-#1(%4*~gia$k2c6W^#LQKYqa94^8dO&E%RHaZf+FW$U)*wmK%Y$>k4??c3xI=4Lj@ ziaO}=?f(530xr5r;4j_0z`LF|vf$csT!Kfs@vg752KCJP)7M7;Zx%lOeq9XH5g+7! z1ouvS(!%j}?!OOD@q5@e09no$zDGR%H114)p>Vui*%qE;+cG-){?!CL?8S%Stqupa z2VYEAU-YZ5I{{Y> (i % 8) + } + for name, v := range map[string]any{ + "pool_base": binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()), + "pool_mask": uint32(1)<<(32-bpfPool.Bits()) - 1, + "pref6": b.prefix.Masked().Addr().As16(), + "mask6": mask, + "v4pos": v4Positions(b.prefix.Bits()), + "slow_mark": uint32(bpfSlowMark), + } { + spec.Variables[name].Set(v) // the program declares each, of this size + } + objs := new(nat64Objects) + if err := spec.LoadAndAssign(objs, nil); err != nil { + return fmt.Errorf("loading the BPF translator: %w", err) + } + b.objs = objs + l, err := link.AttachTCX(link.TCXOptions{Interface: b.ifindex, Program: objs.Nat64, Attach: ebpf.AttachTCXEgress}) + if err != nil { + return fmt.Errorf("attaching the BPF translator, which needs Linux 6.6 or later: %w", err) + } + b.link = l + return nil +} + +// route brings the device up and routes the NAT64 prefix and the pool to it. The device takes no +// address: it never answers for one, and NOARP spares the neighbour lookups. +// +// The pool's route carries an MTU 28 bytes below the smallest LAN's, the IPv6 header's 20 more +// and a fragment header's 8. IPv4 that conntrack reassembled is cut again to it on the way to the +// program, so each piece, translated, still crosses the LAN; a whole packet with DF that would +// not fit earns its sender a Fragmentation Needed from the router. +func (b *bpfNAT64) route(ifi *net.Interface) error { + err := errors.Join(linkSetUp(ifi.Index), routeSet(ifi.Index, b.prefix, netip.Addr{}, 0, 0), route4MTU(ifi.Index, bpfPool, b.lanMTU()-28)) + if err != nil { + return fmt.Errorf("bringing %s up and routing %s and %s to it: %w", ifi.Name, b.prefix, bpfPool, err) + } + return nil +} + +// lanMTU is the smallest MTU of the LANs, 1500 when none can be read. +func (b *bpfNAT64) lanMTU() int { + mtu := 0 + for _, name := range b.lans { + if ifi, err := net.InterfaceByName(name); err == nil && (mtu == 0 || ifi.MTU < mtu) { + mtu = ifi.MTU + } + } + if mtu < 1280 { + return 1500 + } + return mtu +} + +// sweep removes the mappings of pool addresses conntrack has had no flow for over bpfIdleSweeps +// sweeps, so that the pool and the maps do not fill with sources long gone. Without conntrack to +// ask, nothing is removed. +func (b *bpfNAT64) sweep() { + active, err := conntrackSources4(bpfPool) + if err != nil { + return + } + if b.idle == nil { + b.idle = map[[4]byte]int{} + } + var a4 [4]byte + var a6 [16]byte + var gone [][4]byte + it := b.objs.V6Of.Iterate() + for it.Next(&a4, &a6) { + if active[netip.AddrFrom4(a4)] { + delete(b.idle, a4) + continue + } + if b.idle[a4]++; b.idle[a4] >= bpfIdleSweeps { + gone = append(gone, a4) + b.objs.V4Of.Delete(a6) + } + } + for _, a := range gone { + b.objs.V6Of.Delete(a) + delete(b.idle, a) + } +} + +// conntrackSources4 dumps conntrack's IPv4 flows and returns the original sources within pool. +func conntrackSources4(pool netip.Prefix) (map[netip.Addr]bool, error) { + c, err := nfDial() + if err != nil { + return nil, err + } + defer c.Close() + msgs, err := c.Execute(netlink.Message{ + Header: netlink.Header{Type: netlink.HeaderType(unix.NFNL_SUBSYS_CTNETLINK<>4 == 6 { + proto = unix.ETH_P_IPV6 + } + to := &unix.SockaddrLinklayer{Protocol: proto<<8 | proto>>8, Ifindex: p.dev.Index, Halen: 6} + copy(to.Addr[:], p.dev.HardwareAddr) + return unix.Sendto(p.fd, pkt, 0, to) +} + +// serve reads what the program hands over until the reader is closed. +func (p *slowPath) serve(rd *perf.Reader) { + for { + rec, err := rd.Read() + if err != nil { + return + } + p.record(rec) + } +} + +// record takes one record of the perf buffer: a sample, or the count of those it had no room for. +func (p *slowPath) record(rec perf.Record) { + if rec.LostSamples > 0 { + p.dropped.Add(rec.LostSamples) + return + } + p.handle(rec.RawSample) +} + +// handle takes one sample: the frame's length, then the frame. +func (p *slowPath) handle(sample []byte) { + if len(sample) < 4 { + p.dropped.Add(1) + return + } + n := int(binary.NativeEndian.Uint32(sample)) + if n < 14 || len(sample) < 4+n { + p.dropped.Add(1) + return + } + if err := p.packet(sample[4+14 : 4+n]); err != nil { + p.dropped.Add(1) + debugf("[nat64] slow path: %v", err) + } +} + +// packet takes an IP packet, a whole one or a fragment, and sends on what it completes. +func (p *slowPath) packet(pkt []byte) error { + var k slowKey + var hdr, payload []byte + var off int + var more bool + switch { + case len(pkt) >= 20 && pkt[0]>>4 == 4: + tot := int(binary.BigEndian.Uint16(pkt[2:4])) + if pkt[0]&0xf != 5 || tot < 20 || tot > len(pkt) { + return errors.New("malformed IPv4") + } + fo := binary.BigEndian.Uint16(pkt[6:8]) + hdr, payload = pkt[:20], pkt[20:tot] + off, more = int(fo&0x1fff)*8, fo&0x2000 != 0 + k = slowKey{netip.AddrFrom4([4]byte(pkt[12:16])), netip.AddrFrom4([4]byte(pkt[16:20])), uint32(binary.BigEndian.Uint16(pkt[4:6])), pkt[9]} + case len(pkt) >= 40 && pkt[0]>>4 == 6: + end := 40 + int(binary.BigEndian.Uint16(pkt[4:6])) + if end > len(pkt) { + return errors.New("malformed IPv6") + } + hdr, payload = pkt[:40], pkt[40:end] + k = slowKey{netip.AddrFrom16([16]byte(pkt[8:24])), netip.AddrFrom16([16]byte(pkt[24:40])), 0, pkt[6]} + if pkt[6] == unix.IPPROTO_FRAGMENT { + if len(payload) < 8 { + return errors.New("malformed fragment header") + } + fo := binary.BigEndian.Uint16(payload[2:4]) + off, more = int(fo>>3)*8, fo&1 != 0 + k.proto, k.id = payload[0], binary.BigEndian.Uint32(payload[4:8]) + payload = payload[8:] + } + default: + return errors.New("not IP") + } + if off == 0 && !more { + return p.whole(hdr, k.proto, payload) + } + hdr, whole, err := p.reassemble(k, hdr, off, more, payload) + if whole == nil || err != nil { + return err + } + return p.whole(hdr, k.proto, whole) +} + +// reassemble adds a fragment to its datagram, and returns the first fragment's header and the +// payload once all of it is there. Overlapping fragments drop the datagram (RFC 5722). +func (p *slowPath) reassemble(k slowKey, hdr []byte, off int, more bool, data []byte) ([]byte, []byte, error) { + now := p.now() + for key, g := range p.groups { + if now.After(g.expires) { + delete(p.groups, key) + p.dropped.Add(1) + } + } + g := p.groups[k] + if g == nil { + if len(p.groups) >= slowGroups { + return nil, nil, errors.New("too many datagrams being reassembled") + } + g = &slowGroup{parts: map[int][]byte{}, total: -1, expires: now.Add(slowTimeout)} + p.groups[k] = g + } + if off == 0 { + g.hdr = slices.Clone(hdr) + if hdr[0]>>4 == 6 { + g.hdr[6] = k.proto // the header the datagram has without its fragment header + } + } + if !more { + g.total = off + len(data) + } + if (more && len(data)%8 != 0) || off+len(data) > 0xffff || (g.total >= 0 && off+len(data) > g.total) { + delete(p.groups, k) + return nil, nil, errors.New("malformed fragment") + } + for o, d := range g.parts { + if off < o+len(d) && o < off+len(data) { + delete(p.groups, k) + return nil, nil, errors.New("overlapping fragments") + } + } + g.parts[off] = slices.Clone(data) + if g.total < 0 || g.hdr == nil { + return nil, nil, nil + } + whole := make([]byte, 0, g.total) + for _, o := range slices.Sorted(maps.Keys(g.parts)) { + if o != len(whole) { + return nil, nil, nil // a hole yet + } + whole = append(whole, g.parts[o]...) + } + // Whole now: the last piece is among the parts, and nothing lies beyond it + delete(p.groups, k) + return g.hdr, whole, nil +} + +// whole translates a complete datagram and sends it on. +func (p *slowPath) whole(hdr []byte, proto uint8, payload []byte) error { + if hdr[0]>>4 == 4 { + return p.to6(hdr, proto, payload) + } + return p.to4(hdr, proto, payload) +} + +// to6 translates IPv4 from the pool's side: ICMP echo, or UDP whose checksum it computes. +func (p *slowPath) to6(hdr []byte, proto uint8, payload []byte) error { + var dst [16]byte + if err := p.v6of.Lookup([4]byte(hdr[16:20]), &dst); err != nil { + return fmt.Errorf("no source has %s", netip.AddrFrom4([4]byte(hdr[16:20]))) + } + src := embed4(p.prefix, [4]byte(hdr[12:16])) + msg := slices.Clone(payload) + var csum int + switch { + case proto == unix.IPPROTO_ICMP && len(msg) >= 8 && (msg[0] == 8 || msg[0] == 0): + msg[0] = map[byte]byte{8: 128, 0: 129}[msg[0]] + proto, csum = unix.IPPROTO_ICMPV6, 2 + case proto == unix.IPPROTO_UDP && len(msg) >= 8: + csum = 6 + default: + return fmt.Errorf("cannot translate protocol %d whole", proto) + } + msg[csum], msg[csum+1] = 0, 0 + c := l4Checksum(src[:], dst[:], proto, msg) + if c == 0 && proto == unix.IPPROTO_UDP { + c = 0xffff + } + binary.BigEndian.PutUint16(msg[csum:], c) + h := make([]byte, 40) + h[0] = 6<<4 | hdr[1]>>4 + h[1] = hdr[1] << 4 + h[6], h[7] = proto, hdr[8] + copy(h[8:], src[:]) + copy(h[24:], dst[:]) + return p.send6(h, msg) +} + +// to4 translates IPv6 ICMP echo from the LAN side. +func (p *slowPath) to4(hdr []byte, proto uint8, payload []byte) error { + src, dst := [16]byte(hdr[8:24]), [16]byte(hdr[24:40]) + if !p.prefix.Contains(netip.AddrFrom16(dst)) { + return fmt.Errorf("%s is outside %s", netip.AddrFrom16(dst), p.prefix) + } + if proto != unix.IPPROTO_ICMPV6 || len(payload) < 8 || (payload[0] != 128 && payload[0] != 129) { + return fmt.Errorf("cannot translate protocol %d whole", proto) + } + a4, ok := map4(p.v4of, p.v6of, src) + if !ok { + return errors.New("the pool is full") + } + msg := slices.Clone(payload) + msg[0] = map[byte]byte{128: 8, 129: 0}[msg[0]] + msg[2], msg[3] = 0, 0 + binary.BigEndian.PutUint16(msg[2:], l4Checksum(nil, nil, 0, msg)) + h := make([]byte, 20) + h[0] = 4<<4 | 5 + h[1] = hdr[0]<<4 | hdr[1]>>4 + h[8], h[9] = hdr[7], unix.IPPROTO_ICMP + copy(h[12:], a4[:]) + a := extract4(p.prefix, dst) + copy(h[16:], a[:]) + return p.send4(h, msg) +} + +// send4 sends an IPv4 datagram, in fragments when it is larger than the device takes. +func (p *slowPath) send4(h, msg []byte) error { + p.id++ + binary.BigEndian.PutUint16(h[4:6], uint16(p.id)) + chunk := (p.dev.MTU - 20) &^ 7 + for off := 0; off < len(msg); off += chunk { + part := msg[off:min(off+chunk, len(msg))] + fo := off / 8 + if off+len(part) < len(msg) { + fo |= 0x2000 + } + binary.BigEndian.PutUint16(h[2:4], uint16(20+len(part))) + binary.BigEndian.PutUint16(h[6:8], uint16(fo)) + h[10], h[11] = 0, 0 + binary.BigEndian.PutUint16(h[10:], l4Checksum(nil, nil, 0, h)) + if err := p.send(slices.Concat(h, part)); err != nil { + return err + } + } + return nil +} + +// send6 sends an IPv6 datagram, in fragments when it is larger than mtu6. +func (p *slowPath) send6(h, msg []byte) error { + if 40+len(msg) <= p.mtu6 { + binary.BigEndian.PutUint16(h[4:6], uint16(len(msg))) + return p.send(slices.Concat(h, msg)) + } + p.id++ + proto := h[6] + h[6] = unix.IPPROTO_FRAGMENT + chunk := (p.mtu6 - 48) &^ 7 + for off := 0; off < len(msg); off += chunk { + part := msg[off:min(off+chunk, len(msg))] + fo := off + if off+len(part) < len(msg) { + fo |= 1 + } + fh := []byte{proto, 0, byte(fo >> 8), byte(fo), 0, 0, 0, 0} + binary.BigEndian.PutUint32(fh[4:], p.id) + binary.BigEndian.PutUint16(h[4:6], uint16(8+len(part))) + if err := p.send(slices.Concat(h, fh, part)); err != nil { + return err + } + } + return nil +} + +// v4Positions is where RFC 6052 puts the four bytes of an IPv4 address after a prefix of bits: +// right after it, but never in byte 8, the u octet, which stays zero. +func v4Positions(bits int) [4]byte { + var pos [4]byte + for i, b := 0, bits/8; i < 4; b++ { + if b != 8 { + pos[i] = byte(b) + i++ + } + } + return pos +} + +// embed4 is the IPv6 address of an IPv4 one behind the NAT64 prefix (RFC 6052 section 2.2). +func embed4(prefix netip.Prefix, v4 [4]byte) [16]byte { + a := prefix.Masked().Addr().As16() + for i, pos := range v4Positions(prefix.Bits()) { + a[pos] = v4[i] + } + return a +} + +// extract4 is the IPv4 address an address in the NAT64 prefix carries. +func extract4(prefix netip.Prefix, a [16]byte) [4]byte { + var v4 [4]byte + for i, pos := range v4Positions(prefix.Bits()) { + v4[i] = a[pos] + } + return v4 +} + +// l4Checksum is the Internet checksum of msg, after an IPv6 pseudo-header when src is given. +func l4Checksum(src, dst []byte, proto uint8, msg []byte) uint16 { + var sum uint32 + add := func(b []byte) { + for i := 0; i+1 < len(b); i += 2 { + sum += uint32(binary.BigEndian.Uint16(b[i:])) + } + if len(b)%2 == 1 { + sum += uint32(b[len(b)-1]) << 8 + } + } + if src != nil { + add(src) + add(dst) + sum += uint32(len(msg)) + uint32(proto) + } + add(msg) + for sum > 0xffff { + sum = sum>>16 + sum&0xffff + } + return ^uint16(sum) +} + +// map4 is map4 of bpf/nat64.c: the pool address of an IPv6 source, a free one the first time. +func map4(v4of, v6of *ebpf.Map, src [16]byte) ([4]byte, bool) { + var a [4]byte + if v4of.Lookup(src, &a) == nil { + return a, true + } + h := hash6(src) + base := binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()) + mask := uint32(1)<<(32-bpfPool.Bits()) - 1 + for i := range uint32(bpfProbes) { + binary.BigEndian.PutUint32(a[:], base+((h+i)&mask)) + if v6of.Update(a, src, ebpf.UpdateNoExist) != nil { + continue + } + if v4of.Update(src, a, ebpf.UpdateNoExist) == nil { + return a, true + } + v6of.Delete(a) // the program mapped the same source meanwhile + return a, v4of.Lookup(src, &a) == nil + } + return a, false +} + +// hash6 is hash6 of bpf/nat64.c, which reads the address as words in the host's byte order. +func hash6(a [16]byte) uint32 { + w := func(i int) uint32 { return binary.NativeEndian.Uint32(a[4*i:]) } + h := w(0) * 0x9e3779b1 + h = (h ^ w(1)) * 0x85ebca6b + h = (h ^ w(2)) * 0xc2b2ae35 + h = (h ^ w(3)) * 0x27d4eb2f + return h ^ h>>15 +} diff --git a/nat64slow_integration_test.go b/nat64slow_integration_test.go new file mode 100644 index 0000000..a80a234 --- /dev/null +++ b/nat64slow_integration_test.go @@ -0,0 +1,658 @@ +//go:build linux && integration + +package main + +import ( + "bytes" + "encoding/binary" + "errors" + "net" + "net/netip" + "runtime" + "slices" + "strings" + "syscall" + "testing" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" +) + +// slowFixture is a slow path sending through a dummy device of its own, with maps of its own, +// and a socket that sees what leaves the device. +type slowFixture struct { + p *slowPath + cap int +} + +func newSlowFixture(t *testing.T, mtu int) *slowFixture { + t.Helper() + ifi, err := dummyAdd("slowtest0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { linkDel(ifi.Index) }) + if err := linkSetUp(ifi.Index); err != nil { + t.Fatal(err) + } + ifi.MTU = mtu + v4of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 16, ValueSize: 4, MaxEntries: 64}) + if err != nil { + t.Fatal(err) + } + v6of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 4, ValueSize: 16, MaxEntries: 64}) + if err != nil { + t.Fatal(err) + } + objs := &nat64Objects{nat64Maps: nat64Maps{V4Of: v4of, V6Of: v6of}} + t.Cleanup(func() { objs.Close() }) + p, err := newSlowPath(nat64WKP, objs, ifi, 1500) + if err != nil { + t.Fatal(err) + } + t.Cleanup(p.close) + fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_DGRAM, int(htons(unix.ETH_P_ALL))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { unix.Close(fd) }) + if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: ifi.Index}); err != nil { + t.Fatal(err) + } + unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Usec: 200000}) + return &slowFixture{p: p, cap: fd} +} + +// sent returns the IP packets that left the device since the last call. +func (f *slowFixture) sent() [][]byte { + var pkts [][]byte + for { + buf := make([]byte, 2000) + n, _, err := unix.Recvfrom(f.cap, buf, 0) + if err != nil { + return pkts + } + if buf[0]>>4 == 4 || buf[0]>>4 == 6 { + pkts = append(pkts, buf[:n]) + } + } +} + +// v4Packet builds an IPv4 packet whose header checksum is right. +func v4Packet(src, dst string, proto uint8, fo uint16, payload []byte) []byte { + h := make([]byte, 20) + h[0], h[1], h[8], h[9] = 0x45, 0xb8, 64, proto + binary.BigEndian.PutUint16(h[2:], uint16(20+len(payload))) + binary.BigEndian.PutUint16(h[4:], 0x4242) + binary.BigEndian.PutUint16(h[6:], fo) + copy(h[12:], netip.MustParseAddr(src).AsSlice()) + copy(h[16:], netip.MustParseAddr(dst).AsSlice()) + binary.BigEndian.PutUint16(h[10:], l4Checksum(nil, nil, 0, h)) + return append(h, payload...) +} + +func v6Packet(src, dst string, next uint8, payload []byte) []byte { + h := make([]byte, 40) + h[0], h[1], h[6], h[7] = 0x6b, 0x80, next, 64 + binary.BigEndian.PutUint16(h[4:], uint16(len(payload))) + copy(h[8:], netip.MustParseAddr(src).AsSlice()) + copy(h[24:], netip.MustParseAddr(dst).AsSlice()) + return append(h, payload...) +} + +// reassembled puts IPv4 or IPv6 fragments back together, checking each against mtu. +func reassembled(t *testing.T, pkts [][]byte, mtu int) (hdr, payload []byte) { + t.Helper() + whole := map[int][]byte{} + for _, p := range pkts { + if len(p) > mtu { + t.Fatalf("a %d-byte packet is over the MTU of %d", len(p), mtu) + } + if p[0]>>4 == 4 { + hdr = p[:20] + whole[int(binary.BigEndian.Uint16(p[6:])&0x1fff)*8] = p[20:] + continue + } + hdr = p[:40] + off := 0 + data := p[40:] + if p[6] == unix.IPPROTO_FRAGMENT { + off, data = int(binary.BigEndian.Uint16(p[42:])&^7), p[48:] + } + whole[off] = data + } + for _, o := range slices.Sorted(func(yield func(int) bool) { + for o := range whole { + if !yield(o) { + return + } + } + }) { + if o != len(payload) { + t.Fatalf("a hole at %d", len(payload)) + } + payload = append(payload, whole[o]...) + } + return hdr, payload +} + +// The slow path's translations, whole and in fragments, with checksums a receiver accepts. +func TestSlowPathTranslates(t *testing.T) { + if !ownNetns(t) { + return + } + f := newSlowFixture(t, 600) + p := f.p + host := netip.MustParseAddr("2001:db8::5").As16() + pool := [4]byte{198, 18, 0, 5} + if err := p.v6of.Put(pool, host); err != nil { + t.Fatal(err) + } + + // IPv4 UDP without a checksum gets one + udp := []byte{0, 53, 0, 99, 0, 12, 0, 0, 'h', 'e', 'y', '!'} + if err := p.packet(v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_UDP, 0, udp)); err != nil { + t.Fatal(err) + } + pkts := f.sent() + if len(pkts) != 1 { + t.Fatalf("want one packet, got %d", len(pkts)) + } + got := pkts[0] + if src := netip.AddrFrom16([16]byte(got[8:24])); src != netip.MustParseAddr("64:ff9b::c000:202") || [16]byte(got[24:40]) != host || got[6] != unix.IPPROTO_UDP { + t.Fatalf("want UDP from 64:ff9b::c000:202 to 2001:db8::5, got % x", got[:40]) + } + if got[0] != 0x6b || got[1] != 0x80 || got[7] != 64 { + t.Fatalf("the traffic class and hop limit should carry over, got % x", got[:8]) + } + if l4Checksum(got[8:24], got[24:40], unix.IPPROTO_UDP, got[40:]) != 0 { + t.Fatal("the UDP checksum should be right") + } + // A checksum that computes to zero is sent as all ones + zero := slices.Clone(udp) + binary.BigEndian.PutUint16(zero[8:], 0) + binary.BigEndian.PutUint16(zero[10:], 0) + pfx := netip.MustParseAddr("64:ff9b::c000:202").As16() + binary.BigEndian.PutUint16(zero[8:], l4Checksum(pfx[:], host[:], unix.IPPROTO_UDP, zero)) + p.packet(v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_UDP, 0, zero)) + if pkts := f.sent(); len(pkts) != 1 || binary.BigEndian.Uint16(pkts[0][46:]) != 0xffff { + t.Fatalf("want a checksum of 0xffff, got %v", pkts) + } + + // A fragmented echo comes out in fragments for an MTU of 1280 + p.mtu6 = 1280 + echo := append([]byte{8, 0, 0, 0, 0x12, 0x34, 0, 1}, bytes.Repeat([]byte{7}, 2992)...) + binary.BigEndian.PutUint16(echo[2:], l4Checksum(nil, nil, 0, echo)) + for _, part := range []struct { + fo uint16 + data []byte + }{{0x2000 | 1480/8, echo[1480:2960]}, {2960 / 8, echo[2960:]}, {0x2000, echo[:1480]}} { + if err := p.packet(v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_ICMP, part.fo, part.data)); err != nil { + t.Fatal(err) + } + } + hdr, msg := reassembled(t, f.sent(), 1280) + if msg[0] != 128 || len(msg) != len(echo) || l4Checksum(hdr[8:24], hdr[24:40], unix.IPPROTO_ICMPV6, msg) != 0 { + t.Fatalf("want a 3000-byte echo request with a right checksum, got type %d, %d bytes", msg[0], len(msg)) + } + + // An IPv6 echo from a new source gets the pool address the program would give it, and goes + // out in fragments the device takes + reply := append([]byte{129, 0, 0, 0, 0x12, 0x34, 0, 1}, bytes.Repeat([]byte{9}, 1992)...) + src := netip.MustParseAddr("2001:db8::77") + binary.BigEndian.PutUint16(reply[2:], l4Checksum(src.AsSlice(), pfx[:], unix.IPPROTO_ICMPV6, reply)) + frag := func(off int, more bool, data []byte) []byte { + fh := []byte{unix.IPPROTO_ICMPV6, 0, 0, 0, 0, 0, 0, 9} + fo := uint16(off) + if more { + fo |= 1 + } + binary.BigEndian.PutUint16(fh[2:], fo) + return v6Packet(src.String(), "64:ff9b::c000:202", unix.IPPROTO_FRAGMENT, append(fh, data...)) + } + p.packet(frag(0, true, reply[:1024])) + if err := p.packet(frag(1024, false, reply[1024:])); err != nil { + t.Fatal(err) + } + hdr, msg = reassembled(t, f.sent(), 600) + var a4 [4]byte + if err := p.v4of.Lookup(src.As16(), &a4); err != nil { + t.Fatal(err) + } + base, mask := binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()), uint32(1)<<(32-bpfPool.Bits())-1 + if want := base + hash6(src.As16())&mask; binary.BigEndian.Uint32(a4[:]) != want || [4]byte(hdr[12:16]) != a4 { + t.Fatalf("want the source numbered %#x, got %v and a packet from %v", want, a4, hdr[12:16]) + } + if msg[0] != 0 || len(msg) != len(reply) || l4Checksum(nil, nil, 0, msg) != 0 || hdr[1] != 0xb8 || hdr[8] != 64 { + t.Fatalf("want a 2000-byte echo reply with a right checksum, TOS and TTL, got % x", hdr) + } + + // What cannot be translated + for name, pkt := range map[string][]byte{ + "IPv4 to an address no source has": v4Packet("192.0.2.2", "198.18.0.6", unix.IPPROTO_UDP, 0, udp), + "IPv4 TCP": v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_TCP, 0, udp), + "an IPv4 error": v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_ICMP, 0, []byte{3, 3, 0, 0, 0, 0, 0, 0}), + "IPv6 outside the prefix": v6Packet("2001:db8::77", "2001:db8::1", unix.IPPROTO_ICMPV6, reply[:8]), + "IPv6 UDP": v6Packet("2001:db8::77", "64:ff9b::1", unix.IPPROTO_UDP, udp), + "an IPv6 error": v6Packet("2001:db8::77", "64:ff9b::1", unix.IPPROTO_ICMPV6, []byte{1, 4, 0, 0, 0, 0, 0, 0}), + } { + if err := p.packet(pkt); err == nil { + t.Errorf("%s: want an error", name) + } + } + // A source whose every slot is taken + busy := netip.MustParseAddr("2001:db8::99").As16() + for i := range uint32(bpfProbes) { + var slot [4]byte + binary.BigEndian.PutUint32(slot[:], base+((hash6(busy)+i)&mask)) + p.v6of.Put(slot, host) + } + if err := p.packet(v6Packet("2001:db8::99", "64:ff9b::1", unix.IPPROTO_ICMPV6, reply[:8])); err == nil || !strings.Contains(err.Error(), "full") { + t.Fatalf("want the pool full, got %v", err) + } + + // Without the socket nothing goes out, whole or in fragments + unix.Close(p.fd) + p.fd = -1 + p.mtu6 = 1500 + for name, pkt := range map[string][]byte{ + "whole IPv6": v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_UDP, 0, udp), + "fragments IPv6": v4Packet("192.0.2.2", "198.18.0.5", unix.IPPROTO_ICMP, 0, echo), + "IPv4": v6Packet(src.String(), "64:ff9b::1", unix.IPPROTO_ICMPV6, reply[:8]), + } { + if strings.HasPrefix(name, "fragments") { + p.mtu6 = 1280 + } + if err := p.packet(pkt); err == nil { + t.Errorf("%s: want the send refused", name) + } + } +} + +// map4 gives up when the map of sources has no room, after taking back the slot it claimed. +func TestMap4WithoutRoom(t *testing.T) { + if !ownNetns(t) { + return + } + v4of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 16, ValueSize: 4, MaxEntries: 1}) + if err != nil { + t.Fatal(err) + } + defer v4of.Close() + v6of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 4, ValueSize: 16, MaxEntries: 4}) + if err != nil { + t.Fatal(err) + } + defer v6of.Close() + v4of.Put(netip.MustParseAddr("2001:db8::1").As16(), [4]byte{198, 18, 0, 1}) + if _, ok := map4(v4of, v6of, netip.MustParseAddr("2001:db8::2").As16()); ok { + t.Fatal("a full map should refuse a new source") + } + var a [4]byte + var s [16]byte + if v6of.Iterate().Next(&a, &s) { + t.Fatalf("the claimed slot %v should be given back", a) + } +} + +// The program numbers a new source the way map4 does, and passes what the slow path sends back +// to the device's ingress untouched. +func TestBPFProgram(t *testing.T) { + if !ownNetns(t) { + return + } + b := &bpfNAT64{prefix: nat64WKP} + ifi, err := dummyAdd("progtest0") + if err != nil { + t.Fatal(err) + } + defer linkDel(ifi.Index) + b.ifindex = ifi.Index + if err := b.load(); err != nil { + t.Fatal(err) + } + defer b.stop() + frame := func(pkt []byte, proto uint16) []byte { + return append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...) + } + src := netip.MustParseAddr("2001:db8::abcd") + udp := []byte{0, 1, 0, 2, 0, 8, 0x12, 0x34} + ret, err := b.objs.Nat64.Run(&ebpf.RunOptions{Data: frame(v6Packet(src.String(), "64:ff9b::c000:202", unix.IPPROTO_UDP, udp), unix.ETH_P_IPV6)}) + if err != nil || ret != 7 { // TC_ACT_REDIRECT + t.Fatalf("want the packet redirected, got %d: %v", ret, err) + } + var a [4]byte + if err := b.objs.V4Of.Lookup(src.As16(), &a); err != nil { + t.Fatal(err) + } + base, mask := binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()), uint32(1)<<(32-bpfPool.Bits())-1 + if want := base + hash6(src.As16())&mask; binary.BigEndian.Uint32(a[:]) != want { + t.Fatalf("the program numbered %v, map4 would give %#x", a, want) + } + // Attaching to a device that is not there fails, naming what tcx needs + gone := &bpfNAT64{prefix: nat64WKP, ifindex: 1 << 30} + if err := gone.load(); err == nil || !strings.Contains(err.Error(), "6.6") { + t.Fatalf("want the attachment refused, got %v", err) + } + gone.ifindex = 0 + gone.stop() + + // struct __sk_buff from its start to mark, the third word + ctx := []uint32{0, 0, bpfSlowMark} + out := make([]uint32, 48) // the whole struct __sk_buff + ret, err = b.objs.Nat64.Run(&ebpf.RunOptions{Data: frame(make([]byte, 40), unix.ETH_P_IPV6), Context: ctx, ContextOut: out}) + if err != nil || ret != 7 || out[2] != 0 { + t.Fatalf("want a marked packet redirected with its mark cleared, got %d, mark %#x: %v", ret, out[2], err) + } +} + +// Whatever fails part way, start leaves no device behind; given enough descriptors it succeeds. +func TestBPFNAT64StartUndoes(t *testing.T) { + if !ownNetns(t) { + return + } + b := &bpfNAT64{prefix: nat64WKP} + // Once with room, which also gets the runtime's poller its descriptor beforehand + if err := b.start(); err != nil { + t.Fatal(err) + } + b.stop() + for extra := 0; ; extra++ { + if extra > 500 { + t.Fatal("start never succeeded") + } + err := joolWithFDs(t, extra, b.start) + if err == nil { + break + } + if _, err := net.InterfaceByName(joolOutside); err == nil { + t.Fatalf("with room for %d descriptors start failed and left %s", extra, joolOutside) + } + } + b.stop() + // A device of the same name, as a crash would leave, is replaced + if _, err := dummyAdd("reptest0"); err != nil { + t.Fatal(err) + } + ifi, err := dummyAdd("reptest0") + if err != nil { + t.Fatalf("the old device should be replaced: %v", err) + } + linkDel(ifi.Index) + if _, err := net.InterfaceByName(joolOutside); err == nil { + t.Fatalf("%s should be gone", joolOutside) + } +} + +// Without the right to change links, the old device cannot be removed; without the right to ask +// conntrack, the sweep leaves every mapping alone; without descriptors, likewise. +func TestBPFNAT64WithoutPrivileges(t *testing.T) { + if !ownNetns(t) { + return + } + b := &bpfNAT64{prefix: nat64WKP} + if err := b.start(); err != nil { + t.Fatal(err) + } + defer b.stop() + b.objs.V6Of.Put([4]byte{198, 18, 0, 1}, netip.MustParseAddr("2001:db8::1").As16()) + if err := syscall.Setresuid(-1, 65534, -1); err != nil { + t.Fatal(err) + } + _, addErr := dummyAdd(joolOutside) + b.sweep() + b.sweep() + if err := syscall.Setresuid(-1, 0, -1); err != nil { + t.Fatal(err) + } + if addErr == nil || !strings.Contains(addErr.Error(), "removing the old") { + t.Fatalf("want the old device's removal refused, got %v", addErr) + } + joolWithFDs(t, 0, func() error { b.sweep(); b.sweep(); return nil }) + var a [4]byte + var s [16]byte + if !b.objs.V6Of.Iterate().Next(&a, &s) { + t.Fatal("a sweep that cannot ask conntrack should remove nothing") + } + // The slow path's drops are reported with the program's + b.slow.dropped.Add(2) + if d := b.drops(); d != "slow-path 2" { + t.Fatalf("want the slow path's drops, got %q", d) + } +} + +// Without CAP_NET_RAW start gets as far as the slow path's packet socket, and takes down what it +// set up. +func TestBPFNAT64WithoutRawSockets(t *testing.T) { + if !ownNetns(t) { + return + } + b := &bpfNAT64{prefix: nat64WKP} + ch := make(chan error) + go func() { + runtime.LockOSThread() // never unlocked: the thread leaves with its capabilities + hdr := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3} + var data [2]unix.CapUserData + unix.Capget(&hdr, &data[0]) + data[0].Effective &^= 1 << unix.CAP_NET_RAW + if err := unix.Capset(&hdr, &data[0]); err != nil { + ch <- err + return + } + ch <- b.start() + }() + if err := <-ch; err == nil || !strings.Contains(err.Error(), "packet socket") { + t.Fatalf("want the packet socket refused, got %v", err) + } + if _, err := net.InterfaceByName(joolOutside); err == nil { + t.Fatalf("%s should be gone", joolOutside) + } +} + +// Without descriptors there is no packet socket. +func TestSlowPathWithoutDescriptors(t *testing.T) { + lo, err := net.InterfaceByName("lo") + if err != nil { + t.Fatal(err) + } + if err := joolWithFDs(t, 0, func() error { _, err := newSlowPath(nat64WKP, &nat64Objects{}, lo, 1500); return err }); !errors.Is(err, unix.EMFILE) { + t.Fatalf("want the socket refused, got %v", err) + } +} + +// The program's ICMP translation against RFC 7915 sections 4.2 and 5.2, one message at a time: +// type, code and the rest of the header, both checksums, and the quote turned around. +func TestBPFICMPTranslation(t *testing.T) { + if !ownNetns(t) { + return + } + b := &bpfNAT64{prefix: nat64WKP} + ifi, err := dummyAdd("icmptest0") + if err != nil { + t.Fatal(err) + } + b.ifindex = ifi.Index + if err := b.load(); err != nil { + t.Fatal(err) + } + defer b.stop() + host := netip.MustParseAddr("2001:db8::5") + b.objs.V6Of.Put([4]byte{198, 18, 0, 5}, host.As16()) + b.objs.V4Of.Put(host.As16(), [4]byte{198, 18, 0, 5}) + run := func(pkt []byte, proto uint16) ([]byte, bool) { + t.Helper() + out := make([]byte, 2000) + frame := append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...) + opts := &ebpf.RunOptions{Data: frame, DataOut: out} + ret, err := b.objs.Nat64.Run(opts) + if err != nil { + t.Fatal(err) + } + return opts.DataOut[14:], ret == 7 + } + udp := []byte{0, 7, 0x30, 0x39, 0, 8, 0xab, 0xcd} + + // From IPv4: an error from 192.0.2.9 about what 198.18.0.5 sent 192.0.2.2 + quote4 := v4Packet("198.18.0.5", "192.0.2.2", unix.IPPROTO_UDP, 0x4000, udp) + for _, c := range []struct { + typ, code byte + rest uint32 + want []uint32 // type, code, rest; none to be dropped + }{ + {3, 0, 0, []uint32{1, 0, 0}}, + {3, 1, 0, []uint32{1, 0, 0}}, + {3, 2, 0, []uint32{4, 1, 6}}, + {3, 3, 0, []uint32{1, 4, 0}}, + {3, 4, 1400, []uint32{2, 0, 1420}}, + {3, 4, 1000, []uint32{2, 0, 1280}}, + {3, 4, 0, []uint32{2, 0, 1280}}, + {3, 5, 0, []uint32{1, 0, 0}}, + {3, 6, 0, []uint32{1, 0, 0}}, + {3, 7, 0, []uint32{1, 0, 0}}, + {3, 8, 0, []uint32{1, 0, 0}}, + {3, 9, 0, []uint32{1, 1, 0}}, + {3, 10, 0, []uint32{1, 1, 0}}, + {3, 11, 0, []uint32{1, 0, 0}}, + {3, 12, 0, []uint32{1, 0, 0}}, + {3, 13, 0, []uint32{1, 1, 0}}, + {3, 14, 0, nil}, + {3, 15, 0, []uint32{1, 1, 0}}, + {3, 16, 0, nil}, + {11, 0, 0, []uint32{3, 0, 0}}, + {11, 1, 0, []uint32{3, 1, 0}}, + {12, 0, 0 << 24, []uint32{4, 0, 0}}, + {12, 0, 1 << 24, []uint32{4, 0, 1}}, + {12, 0, 2 << 24, []uint32{4, 0, 4}}, + {12, 0, 3 << 24, []uint32{4, 0, 4}}, + {12, 0, 4 << 24, nil}, + {12, 0, 8 << 24, []uint32{4, 0, 7}}, + {12, 0, 9 << 24, []uint32{4, 0, 6}}, + {12, 0, 10 << 24, nil}, + {12, 2, 13 << 24, []uint32{4, 0, 8}}, + {12, 0, 17 << 24, []uint32{4, 0, 24}}, + {12, 0, 20 << 24, nil}, + {12, 1, 0, nil}, + {5, 0, 0, nil}, // Redirect + {13, 0, 0, nil}, // Timestamp + } { + msg := binary.BigEndian.AppendUint32([]byte{c.typ, c.code, 0, 0}, c.rest) + msg = append(msg, quote4...) + binary.BigEndian.PutUint16(msg[2:], l4Checksum(nil, nil, 0, msg)) + got, ok := run(v4Packet("192.0.2.9", "198.18.0.5", unix.IPPROTO_ICMP, 0, msg), unix.ETH_P_IP) + if ok != (c.want != nil) { + t.Errorf("ICMP %d/%d: translated %v, want %v", c.typ, c.code, ok, c.want != nil) + continue + } + if !ok { + continue + } + icmp := got[40:] + if g := []uint32{uint32(icmp[0]), uint32(icmp[1]), binary.BigEndian.Uint32(icmp[4:])}; !slices.Equal(g, c.want) { + t.Errorf("ICMP %d/%d: got %v, want %v", c.typ, c.code, g, c.want) + } + if l4Checksum(got[8:24], got[24:40], unix.IPPROTO_ICMPV6, icmp) != 0 { + t.Errorf("ICMP %d/%d: the ICMPv6 checksum is wrong", c.typ, c.code) + } + if q := icmp[8:]; [16]byte(q[8:24]) != host.As16() || netip.AddrFrom16([16]byte(q[24:40])) != netip.MustParseAddr("64:ff9b::c000:202") || q[6] != unix.IPPROTO_UDP || binary.BigEndian.Uint16(q[4:]) != 8 || !bytes.Equal(q[40:48], udp) { + t.Errorf("ICMP %d/%d: the quote should be the UDP from 2001:db8::5 to 64:ff9b::c000:202, got % x", c.typ, c.code, q) + } + } + + // From IPv6: an error from 2001:db8::5 about what 64:ff9b::c000:209 sent it + quote6 := v6Packet("64:ff9b::c000:209", host.String(), unix.IPPROTO_UDP, udp) + src, dst := host.As16(), netip.MustParseAddr("64:ff9b::c000:209").As16() + for _, c := range []struct { + typ, code byte + rest uint32 + want []uint32 + }{ + {1, 0, 0, []uint32{3, 1, 0}}, + {1, 1, 0, []uint32{3, 10, 0}}, + {1, 2, 0, []uint32{3, 1, 0}}, + {1, 3, 0, []uint32{3, 1, 0}}, + {1, 4, 0, []uint32{3, 3, 0}}, + {1, 5, 0, nil}, + {2, 0, 1500, []uint32{3, 4, 1480}}, + {2, 0, 70, []uint32{3, 4, 68}}, + {2, 0, 100000, []uint32{3, 4, 0xffff}}, + {3, 0, 0, []uint32{11, 0, 0}}, + {3, 1, 0, []uint32{11, 1, 0}}, + {4, 0, 0, []uint32{12, 0, 0}}, + {4, 0, 1, []uint32{12, 0, 1 << 24}}, + {4, 0, 2, nil}, + {4, 0, 4, []uint32{12, 0, 2 << 24}}, + {4, 0, 6, []uint32{12, 0, 9 << 24}}, + {4, 0, 7, []uint32{12, 0, 8 << 24}}, + {4, 0, 9, []uint32{12, 0, 12 << 24}}, + {4, 0, 30, []uint32{12, 0, 16 << 24}}, + {4, 0, 40, nil}, + {4, 1, 0, []uint32{3, 2, 0}}, + {4, 2, 0, nil}, + {133, 0, 0, nil}, // Router Solicitation + } { + msg := binary.BigEndian.AppendUint32([]byte{c.typ, c.code, 0, 0}, c.rest) + msg = append(msg, quote6...) + binary.BigEndian.PutUint16(msg[2:], l4Checksum(src[:], dst[:], unix.IPPROTO_ICMPV6, msg)) + got, ok := run(v6Packet(host.String(), "64:ff9b::c000:209", unix.IPPROTO_ICMPV6, msg), unix.ETH_P_IPV6) + if ok != (c.want != nil) { + t.Errorf("ICMPv6 %d/%d: translated %v, want %v", c.typ, c.code, ok, c.want != nil) + continue + } + if !ok { + continue + } + icmp := got[20:binary.BigEndian.Uint16(got[2:])] + if g := []uint32{uint32(icmp[0]), uint32(icmp[1]), binary.BigEndian.Uint32(icmp[4:])}; !slices.Equal(g, c.want) { + t.Errorf("ICMPv6 %d/%d: got %v, want %v", c.typ, c.code, g, c.want) + } + if l4Checksum(nil, nil, 0, icmp) != 0 || l4Checksum(nil, nil, 0, got[:20]) != 0 { + t.Errorf("ICMPv6 %d/%d: a checksum is wrong", c.typ, c.code) + } + if q := icmp[8:]; l4Checksum(nil, nil, 0, q[:20]) != 0 || [4]byte(q[12:16]) != [4]byte{192, 0, 2, 9} || [4]byte(q[16:20]) != [4]byte{198, 18, 0, 5} || !bytes.Equal(q[20:28], udp) { + t.Errorf("ICMPv6 %d/%d: the quote should be the UDP from 192.0.2.9 to 198.18.0.5, got % x", c.typ, c.code, q) + } + } +} + +// The program embeds and extracts IPv4 addresses the way RFC 6052 does behind a prefix of every +// length, and leaves alone what is outside the prefix. +func TestBPFRFC6052(t *testing.T) { + if !ownNetns(t) { + return + } + host := netip.MustParseAddr("2001:db8::5") + udp := []byte{0, 7, 0x30, 0x39, 0, 8, 0xab, 0xcd} + for prefix, want := range rfc6052Examples { + b := &bpfNAT64{prefix: netip.MustParsePrefix(prefix)} + ifi, err := dummyAdd("rfctest0") + if err != nil { + t.Fatal(err) + } + b.ifindex = ifi.Index + if err := b.load(); err != nil { + t.Fatal(err) + } + b.objs.V6Of.Put([4]byte{198, 18, 0, 5}, host.As16()) + run := func(pkt []byte, proto uint16) ([]byte, bool) { + out := make([]byte, 200) + opts := &ebpf.RunOptions{Data: append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...), DataOut: out} + ret, err := b.objs.Nat64.Run(opts) + if err != nil { + t.Fatal(err) + } + return opts.DataOut[14:], ret == 7 + } + got, ok := run(v4Packet("192.0.2.33", "198.18.0.5", unix.IPPROTO_UDP, 0, udp), unix.ETH_P_IP) + if src := netip.AddrFrom16([16]byte(got[8:24])); !ok || src != netip.MustParseAddr(want) { + t.Errorf("%s: 192.0.2.33 became %s, want %s", prefix, src, want) + } + got, ok = run(v6Packet(host.String(), want, unix.IPPROTO_UDP, udp), unix.ETH_P_IPV6) + if dst := netip.AddrFrom4([4]byte(got[16:20])); !ok || dst != netip.MustParseAddr("192.0.2.33") { + t.Errorf("%s: %s became %s, want 192.0.2.33", prefix, want, dst) + } + if _, ok := run(v6Packet(host.String(), "2001:db9::1", unix.IPPROTO_UDP, udp), unix.ETH_P_IPV6); ok { + t.Errorf("%s: an address outside the prefix should not be translated", prefix) + } + b.stop() + } +} diff --git a/nat64slow_test.go b/nat64slow_test.go new file mode 100644 index 0000000..c735b82 --- /dev/null +++ b/nat64slow_test.go @@ -0,0 +1,189 @@ +//go:build linux + +package main + +import ( + "encoding/binary" + "net/netip" + "testing" + "time" + + "github.com/cilium/ebpf/perf" + "github.com/mdlayher/netlink" +) + +// A sample too short for its length or for an Ethernet frame is dropped, and so is a record of +// samples the buffer had no room for. +func TestSlowPathRecords(t *testing.T) { + p := &slowPath{} + sample := func(n int, frame []byte) []byte { + return append(binary.NativeEndian.AppendUint32(nil, uint32(n)), frame...) + } + for _, rec := range []perf.Record{ + {RawSample: []byte{1, 2}}, + {RawSample: sample(10, make([]byte, 10))}, // shorter than an Ethernet header + {RawSample: sample(100, make([]byte, 20))}, // shorter than it says + {RawSample: sample(20, make([]byte, 20))}, // no IP after the Ethernet header + {LostSamples: 3}, + } { + p.record(rec) + } + if n := p.dropped.Load(); n != 7 { + t.Fatalf("want 4 bad samples and 3 lost counted, got %d", n) + } +} + +// What is not a well-formed IPv4 or IPv6 packet is refused before anything is reassembled. +func TestSlowPathRefusesMalformed(t *testing.T) { + p := &slowPath{groups: map[slowKey]*slowGroup{}, now: time.Now} + v4 := func(ihl byte, tot uint16, n int) []byte { + b := make([]byte, n) + b[0] = 4<<4 | ihl + binary.BigEndian.PutUint16(b[2:], tot) + return b + } + v6 := func(plen uint16, next byte, n int) []byte { + b := make([]byte, n) + b[0], b[6] = 6<<4, next + binary.BigEndian.PutUint16(b[4:], plen) + return b + } + for name, pkt := range map[string][]byte{ + "empty": nil, + "IPv4 options": v4(6, 24, 24), + "IPv4 length short": v4(5, 10, 20), + "IPv4 length long": v4(5, 40, 20), + "IPv6 length long": v6(10, 17, 40), + "IPv6 fragment short": v6(4, 44, 44), + } { + if err := p.packet(pkt); err == nil { + t.Errorf("%s: want an error", name) + } + } +} + +// Fragments come together in any order; a hole waits, an overlap or a malformed piece drops the +// datagram, and so do too many datagrams at once or one that takes too long. +func TestSlowPathReassembly(t *testing.T) { + now := time.Unix(1000, 0) + p := &slowPath{groups: map[slowKey]*slowGroup{}, now: func() time.Time { return now }} + hdr := make([]byte, 40) + hdr[0], hdr[6] = 6<<4, 44 + key := func(id uint32) slowKey { + return slowKey{netip.MustParseAddr("2001:db8::1"), netip.MustParseAddr("64:ff9b::1"), id, 58} + } + add := func(id uint32, off int, more bool, n int) ([]byte, []byte, error) { + return p.reassemble(key(id), hdr, off, more, make([]byte, n)) + } + + if _, w, err := add(1, 16, false, 4); w != nil || err != nil { + t.Fatalf("the last piece alone completes nothing: %v %v", w, err) + } + if _, w, err := add(1, 0, true, 8); w != nil || err != nil { + t.Fatalf("a hole is left at 8: %v %v", w, err) + } + h, w, err := add(1, 8, true, 8) + if err != nil || len(w) != 20 || h[6] != 58 { + t.Fatalf("want 20 bytes and the header naming ICMPv6, got %d, %v, %v", len(w), h, err) + } + if len(p.groups) != 0 { + t.Fatal("a datagram put together should be forgotten") + } + + for name, step := range map[string]func() error{ + "a piece with more that is not a multiple of 8": func() error { _, _, err := add(2, 0, true, 7); return err }, + "a piece beyond 65535": func() error { _, _, err := add(3, 65528, false, 16); return err }, + "a piece beyond the end": func() error { + add(4, 16, false, 8) + _, _, err := add(4, 16, true, 16) + return err + }, + "overlapping pieces": func() error { + add(5, 0, true, 16) + _, _, err := add(5, 8, true, 16) + return err + }, + } { + if err := step(); err == nil { + t.Errorf("%s: want an error", name) + } + } + if len(p.groups) != 0 { + t.Fatalf("each bad datagram should be dropped, %d left", len(p.groups)) + } + + for id := range uint32(slowGroups) { + add(100+id, 0, true, 8) + } + if _, _, err := add(999, 0, true, 8); err == nil { + t.Fatal("a datagram over the limit should be refused") + } + dropped := p.dropped.Load() + now = now.Add(slowTimeout + time.Second) + if _, _, err := add(999, 0, true, 8); err != nil { + t.Fatalf("the expired datagrams should make room: %v", err) + } + if n := p.dropped.Load() - dropped; n != slowGroups { + t.Fatalf("want the %d expired datagrams counted, got %d", slowGroups, n) + } +} + +// Only the original sources within the pool count, and what does not parse is passed over. +func TestPoolSources(t *testing.T) { + flow := func(src [4]byte) netlink.Message { + ae := netlink.NewAttributeEncoder() + ae.Nested(ctaTupleOrig, func(ae *netlink.AttributeEncoder) error { + ae.Uint8(9, 0) // not the IP tuple + ae.Nested(ctaTupleIP, func(ae *netlink.AttributeEncoder) error { + ae.Bytes(1, src[:]) + ae.Bytes(2, []byte{192, 0, 2, 2}) + return nil + }) + return nil + }) + ae.Nested(ctaTupleReply, func(ae *netlink.AttributeEncoder) error { + ae.Nested(ctaTupleIP, func(ae *netlink.AttributeEncoder) error { + ae.Bytes(1, []byte{198, 18, 9, 9}) + return nil + }) + return nil + }) + b, _ := ae.Encode() + return netlink.Message{Data: append(make([]byte, nfgenmsgLen), b...)} + } + got := poolSources([]netlink.Message{ + flow([4]byte{198, 18, 0, 1}), + flow([4]byte{10, 0, 0, 1}), + {Data: []byte{0, 0}}, // shorter than nfgenmsg + {Data: []byte{0, 0, 0, 0, 9, 0}}, // an attribute cut short + }, bpfPool) + if len(got) != 1 || !got[netip.MustParseAddr("198.18.0.1")] { + t.Fatalf("want only 198.18.0.1, got %v", got) + } +} + +const ctaTupleReply = 2 // CTA_TUPLE_REPLY + +// The examples of RFC 6052 section 2.4: 192.0.2.33 behind a prefix of each length. +var rfc6052Examples = map[string]string{ + "2001:db8::/32": "2001:db8:c000:221::", + "2001:db8:100::/40": "2001:db8:1c0:2:21::", + "2001:db8:122::/48": "2001:db8:122:c000:2:2100::", + "2001:db8:122:300::/56": "2001:db8:122:3c0:0:221::", + "2001:db8:122:344::/64": "2001:db8:122:344:c0:2:2100:0", + "2001:db8:122:344::/96": "2001:db8:122:344::c000:221", +} + +func TestRFC6052(t *testing.T) { + v4 := [4]byte{192, 0, 2, 33} + for prefix, want := range rfc6052Examples { + p := netip.MustParsePrefix(prefix) + a := embed4(p, v4) + if got := netip.AddrFrom16(a); got != netip.MustParseAddr(want) { + t.Errorf("%s: embedded %s, want %s", prefix, got, want) + } + if got := extract4(p, a); got != v4 { + t.Errorf("%s: extracted %v", prefix, got) + } + } +} From 0c3ff62e20e21ddd281da7efd386f50f6f5bd734 Mon Sep 17 00:00:00 2001 From: Qishuai Liu Date: Sun, 11 Oct 2026 23:45:50 +0900 Subject: [PATCH 2/2] Load the BPF translator without cilium/ebpf, and recommend it over Jool cilium/ebpf grew the binary by 1.6 MB, past the 6 MiB limit, and its perf package does not build for 32-bit big-endian MIPS. tools/bpfgen now compiles bpf/nat64.c for either byte order at go generate time and writes the instructions, maps, read-only data and relocations, and bpf.go loads them with the bpf() system call alone. The slow path reads a BPF ring buffer instead of a perf buffer. The binary is now 98 KB over main's. The recipes and the standards document recommend -nat64 bpf, with Jool for kernels older than 6.6, and sixup refuses to start when the pool 198.18.0.0/15 overlaps an address or route of the host, as it does for Jool's /31. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 2 +- bpf.go | 321 ++++++++++++++++++++++++++++++++ bpf/nat64.c | 71 +++---- bpf_integration_test.go | 54 ++++++ docs/recipes.ja.md | 14 +- docs/recipes.md | 31 ++-- docs/recipes.zh.md | 14 +- docs/standards.md | 6 +- flags.go | 4 +- go.mod | 1 - go.sum | 5 - jool.go | 11 +- jool_integration_test.go | 10 +- jool_stub.go | 2 +- main.go | 11 +- nat64_bpf.go | 38 ++++ nat64_bpfeb.bin | Bin 0 -> 20376 bytes nat64_bpfeb.go | 339 +++++++++++++++------------------- nat64_bpfeb.o | Bin 70672 -> 0 bytes nat64_bpfel.bin | Bin 0 -> 20664 bytes nat64_bpfel.go | 333 +++++++++++++++------------------ nat64_bpfel.o | Bin 71648 -> 0 bytes nat64bpf.go | 82 ++++---- nat64bpf_integration_test.go | 10 +- nat64slow.go | 52 ++---- nat64slow_integration_test.go | 129 ++++++------- nat64slow_test.go | 24 +-- tools/bpfgen/main.go | 238 ++++++++++++++++++++++++ 28 files changed, 1177 insertions(+), 625 deletions(-) create mode 100644 bpf.go create mode 100644 bpf_integration_test.go create mode 100644 nat64_bpf.go create mode 100644 nat64_bpfeb.bin delete mode 100644 nat64_bpfeb.o create mode 100644 nat64_bpfel.bin delete mode 100644 nat64_bpfel.o create mode 100644 tools/bpfgen/main.go diff --git a/Dockerfile b/Dockerfile index e2f96e7..7826e23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG TARGETARCH TARGETVARIANT VERSION=dev WORKDIR /src COPY go.mod go.sum ./ RUN go mod download -COPY *.go *.o LICENSE NOTICE ./ +COPY *.go *.bin LICENSE NOTICE ./ RUN CGO_ENABLED=0 GOOS=linux GOARCH="$TARGETARCH" GOARM="${TARGETVARIANT#v}" \ go build -trimpath -ldflags="-s -w -X main.version=${VERSION}" -o /out/sixup . diff --git a/bpf.go b/bpf.go new file mode 100644 index 0000000..3b69e18 --- /dev/null +++ b/bpf.go @@ -0,0 +1,321 @@ +//go:build linux + +package main + +import ( + "bytes" + "errors" + "fmt" + "os" + "runtime" + "strconv" + "strings" + "sync/atomic" + "unsafe" + + "golang.org/x/sys/unix" +) + +// A small loader for the programs tools/bpfgen prepares: the bpf() system call and nothing else, +// so that sixup carries no ELF or BTF parser. The attribute layouts are those of union bpf_attr +// in linux/bpf.h. + +type bpfMapSpec struct { + name string + typ, keySize, valueSize, maxEntries, flags uint32 +} + +// bpfReloc points a 64-bit load of the code at a map, or at an offset into the read-only data. +type bpfReloc struct { + insn, m int + off int64 // -1 for the map itself +} + +type bpfVar struct{ off, size int } + +// bpfProgram is what tools/bpfgen writes for one program, with the read-only data filled in. +type bpfProgram struct { + code []byte + maps []bpfMapSpec + relocs []bpfReloc + rodata []byte + license string +} + +// bpfObjects is a program loaded with its maps. +type bpfObjects struct { + maps []*bpfMap + prog int +} + +// srcShift is where the source register sits in the second byte of an instruction: its high +// nibble on a little-endian machine, its low one on a big-endian one. +var srcShift = 4 * (nativeEndian.Uint16([]byte{1, 0}) & 1) + +func bpfSys(cmd int, attr unsafe.Pointer, size uintptr) (int, error) { + r, _, errno := unix.Syscall(unix.SYS_BPF, uintptr(cmd), uintptr(attr), size) + runtime.KeepAlive(attr) + if errno != 0 { + return -1, errno + } + return int(r), nil +} + +func ptr(b []byte) uint64 { + if len(b) == 0 { + return 0 + } + return uint64(uintptr(unsafe.Pointer(&b[0]))) +} + +type bpfMap struct { + fd int + spec bpfMapSpec + valueSize int // per CPU too for the per-CPU ones +} + +func bpfMapCreate(spec bpfMapSpec) (*bpfMap, error) { + attr := struct { + typ, keySize, valueSize, maxEntries, flags, innerMapFd, numaNode uint32 + name [16]byte + }{typ: spec.typ, keySize: spec.keySize, valueSize: spec.valueSize, maxEntries: spec.maxEntries, flags: spec.flags} + copy(attr.name[:15], strings.TrimPrefix(spec.name, ".")) + fd, err := bpfSys(unix.BPF_MAP_CREATE, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) + if err != nil { + return nil, fmt.Errorf("creating the map %s: %w", spec.name, err) + } + m := &bpfMap{fd: fd, spec: spec, valueSize: int(spec.valueSize)} + if spec.typ == unix.BPF_MAP_TYPE_PERCPU_ARRAY || spec.typ == unix.BPF_MAP_TYPE_PERCPU_HASH { + m.valueSize = ((m.valueSize + 7) &^ 7) * possibleCPUs() // each CPU's rounded up to 8 bytes + } + return m, nil +} + +func (m *bpfMap) close() { unix.Close(m.fd) } + +// elem is one of the map element commands, with key, and value or next key. +func (m *bpfMap) elem(cmd int, key, value []byte, flags uint64) error { + attr := struct { + fd, _ uint32 + key, value uint64 + flags uint64 + }{fd: uint32(m.fd), key: ptr(key), value: ptr(value), flags: flags} + _, err := bpfSys(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) + runtime.KeepAlive(key) + runtime.KeepAlive(value) + return err +} + +// lookup fills value, which for a per-CPU map holds every CPU's, and fails with ENOENT for a key +// the map does not have. +func (m *bpfMap) lookup(key, value []byte) error { + return m.elem(unix.BPF_MAP_LOOKUP_ELEM, key, value, 0) +} + +func (m *bpfMap) update(key, value []byte, flags uint64) error { + return m.elem(unix.BPF_MAP_UPDATE_ELEM, key, value, flags) +} + +func (m *bpfMap) delete(key []byte) error { return m.elem(unix.BPF_MAP_DELETE_ELEM, key, nil, 0) } + +// keys returns the keys in the map, as many as it lists in one pass. +func (m *bpfMap) keys() [][]byte { + var keys [][]byte + var key []byte // nil asks for the first + for { + next := make([]byte, m.spec.keySize) + if m.elem(unix.BPF_MAP_GET_NEXT_KEY, key, next, 0) != nil { + return keys + } + keys = append(keys, next) + key = next + } +} + +// possibleCPUs counts the CPUs a per-CPU map has a value for: /sys/devices/system/cpu/possible, +// such as 0-3 or 0,2-5. +func possibleCPUs() int { + b, _ := os.ReadFile("/sys/devices/system/cpu/possible") // sysfs is there wherever BPF is + n := 0 + for r := range strings.SplitSeq(strings.TrimSpace(string(b)), ",") { + lo, hi, ok := strings.Cut(r, "-") + first, _ := strconv.Atoi(lo) + last := first + if ok { + last, _ = strconv.Atoi(hi) + } + n += last - first + 1 + } + return n +} + +// bpfLoad creates the maps, fills in and freezes the read-only data, points the code at both +// and loads it as a tc program. A program the verifier refuses comes back with the end of its log. +func bpfLoad(p bpfProgram) (_ *bpfObjects, err error) { + objs := &bpfObjects{prog: -1} + defer func() { + if err != nil { + objs.close() + } + }() + for _, spec := range p.maps { + m, err := bpfMapCreate(spec) + if err != nil { + return nil, err + } + objs.maps = append(objs.maps, m) + } + ro := objs.maps[len(objs.maps)-1] + freeze := func() error { + attr := struct{ fd uint32 }{uint32(ro.fd)} + _, err := bpfSys(unix.BPF_MAP_FREEZE, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) + return err + } + if err := errors.Join(ro.update(make([]byte, 4), p.rodata, 0), freeze()); err != nil { + return nil, fmt.Errorf("filling in and freezing the read-only data: %w", err) + } + + code := append([]byte(nil), p.code...) + for _, r := range p.relocs { + insn := code[r.insn*8:] + src := byte(unix.BPF_PSEUDO_MAP_FD) + if r.off >= 0 { + src = unix.BPF_PSEUDO_MAP_VALUE + nativeEndian.PutUint32(insn[12:], uint32(r.off)) // the second half of the 64-bit load + } + insn[1] = insn[1]&^(0xf<= 0 { + unix.Close(o.prog) + } +} + +// attachTCX attaches the program to the egress of a device; closing the descriptor it returns +// detaches it, and so does sixup ending. +func (o *bpfObjects) attachTCX(ifindex int) (int, error) { + attr := struct { + prog, ifindex, attachType, flags uint32 + relative, _ uint32 + expectedRevision uint64 + }{prog: uint32(o.prog), ifindex: uint32(ifindex), attachType: unix.BPF_TCX_EGRESS} + return bpfSys(unix.BPF_LINK_CREATE, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) +} + +// errRingClosed is what a ring buffer reader returns once closed. +var errRingClosed = errors.New("ring buffer closed") + +// ringReader reads the records of a BPF ring buffer: the consumer position on a page it writes, +// the producer position on the next, and the data mapped twice after it, so that a record that +// wraps around reads straight through. An eventfd in the same epoll set wakes it to stop. +type ringReader struct { + cons, prod []byte + mask uint64 + epfd, evfd int +} + +func newRingReader(m *bpfMap) (_ *ringReader, err error) { + page := os.Getpagesize() + r := &ringReader{mask: uint64(m.spec.maxEntries) - 1, epfd: -1, evfd: -1} + defer func() { + if err != nil { + r.free() + } + }() + if r.cons, err = unix.Mmap(m.fd, 0, page, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED); err != nil { + return nil, err + } + if r.prod, err = unix.Mmap(m.fd, int64(page), page+2*int(m.spec.maxEntries), unix.PROT_READ, unix.MAP_SHARED); err != nil { + return nil, err + } + if r.epfd, err = unix.EpollCreate1(unix.EPOLL_CLOEXEC); err != nil { + return nil, err + } + if r.evfd, err = unix.Eventfd(0, unix.EFD_CLOEXEC); err != nil { + return nil, err + } + for _, fd := range []int{m.fd, r.evfd} { + // A new descriptor in a new set, which fails only for want of memory + unix.EpollCtl(r.epfd, unix.EPOLL_CTL_ADD, fd, &unix.EpollEvent{Events: unix.EPOLLIN, Fd: int32(fd)}) + } + return r, nil +} + +// read returns the next record, waiting for one, or errRingClosed once close is called. +func (r *ringReader) read() ([]byte, error) { + page := uint64(os.Getpagesize()) + cons := (*uint64)(unsafe.Pointer(&r.cons[0])) + prod := (*uint64)(unsafe.Pointer(&r.prod[0])) + events := make([]unix.EpollEvent, 2) + for { + // A record the program is still writing is waited for like none: committing it wakes + // the reader that has not consumed it. The program discards none. + if c := atomic.LoadUint64(cons); c < atomic.LoadUint64(prod) { + off := page + c&r.mask + if hdr := atomic.LoadUint32((*uint32)(unsafe.Pointer(&r.prod[off]))); hdr&unix.BPF_RINGBUF_BUSY_BIT == 0 { + n := uint64(hdr) + rec := append([]byte(nil), r.prod[off+unix.BPF_RINGBUF_HDR_SZ:][:n]...) + atomic.StoreUint64(cons, c+(n+unix.BPF_RINGBUF_HDR_SZ+7)&^7) + return rec, nil + } + } + n, err := unix.EpollWait(r.epfd, events, -1) + if errors.Is(err, unix.EINTR) { + continue + } + if err != nil { + return nil, err + } + for _, e := range events[:n] { + if int(e.Fd) == r.evfd { + return nil, errRingClosed + } + } + } +} + +// close makes read return; free releases the reader once nothing reads. +func (r *ringReader) close() { unix.Write(r.evfd, []byte{1, 0, 0, 0, 0, 0, 0, 0}) } + +func (r *ringReader) free() { + for _, b := range [][]byte{r.cons, r.prod} { + if b != nil { + unix.Munmap(b) + } + } + for _, fd := range []int{r.epfd, r.evfd} { + if fd >= 0 { + unix.Close(fd) + } + } +} diff --git a/bpf/nat64.c b/bpf/nat64.c index f70dfc4..9dbf1ae 100644 --- a/bpf/nat64.c +++ b/bpf/nat64.c @@ -9,7 +9,7 @@ // TCP, UDP and ICMP are translated, ICMP errors with the packet they quote, and fragments one by // one. Fragments reach the program as they were sent, or as conntrack's reassembly is cut again // on the way out of the device, no larger than the pool route's MTU leaves room for. Two kinds -// need the whole datagram for their new checksum and go to sixup instead, through the perf buffer +// need the whole datagram for their new checksum and go to sixup instead, through the ring buffer // slow: fragmented ICMP, whose IPv6 checksum covers a length no single fragment shows, and IPv4 // UDP without a checksum, which IPv6 requires. sixup sends them back translated through the same // device, marked, and the program hands them to its ingress like the rest. Dropped and counted: IPv4 options, IPv6 extension @@ -46,6 +46,16 @@ volatile const __u8 mask6[16]; // ones over the prefix's length volatile const __u8 v4pos[4]; // where RFC 6052 puts the bytes of an IPv4 address after it volatile const __u32 slow_mark; // marks the packets sixup sends back translated +// A map as sixup creates it: tools/bpfgen reads these from the section maps, so that sixup needs +// neither BTF nor an ELF parser +struct map_def { + __u32 type; + __u32 key_size; + __u32 value_size; + __u32 max_entries; + __u32 flags; +}; + enum { STAT_OUT, // IPv6 translated to IPv4 STAT_IN, // IPv4 translated to IPv6 @@ -58,35 +68,24 @@ enum { STAT_MAX, }; -struct { - __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); - __uint(max_entries, STAT_MAX); - __type(key, __u32); - __type(value, __u64); -} stats SEC(".maps"); +struct map_def stats SEC("maps") = {BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(__u32), sizeof(__u64), STAT_MAX}; // The two directions of the address mapping, kept in step; sixup removes the pairs conntrack has // no flow for -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 65536); - __type(key, struct in6_addr); - __type(value, __be32); -} v4_of SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 65536); - __type(key, __be32); - __type(value, struct in6_addr); -} v6_of SEC(".maps"); - -// The packets handed to sixup, each after its length -struct { - __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); - __uint(key_size, sizeof(__u32)); - __uint(value_size, sizeof(__u32)); -} slow SEC(".maps"); +struct map_def v4_of SEC("maps") = {BPF_MAP_TYPE_HASH, sizeof(struct in6_addr), sizeof(__be32), 65536}; +struct map_def v6_of SEC("maps") = {BPF_MAP_TYPE_HASH, sizeof(__be32), sizeof(struct in6_addr), 65536}; + +#define SLOW_MAX 16384 // the largest frame handed to sixup, more than a jumbo LAN's; a power of two + +// The frames handed to sixup +struct map_def slow SEC("maps") = {BPF_MAP_TYPE_RINGBUF, 0, 0, 1 << 22}; + +// Where a frame is copied on its way there, as a ring buffer takes no packet directly +struct slow_frame { + __u8 b[SLOW_MAX]; +}; + +struct map_def slow_buf SEC("maps") = {BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(__u32), sizeof(struct slow_frame), 1}; // The IPv4 UDP datagrams without a checksum whose first fragment went to sixup, so that the rest // follows it there: the kernel cuts a reassembled datagram again first fragment first @@ -97,12 +96,7 @@ struct frag4 { __u16 pad; }; -struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __uint(max_entries, 1024); - __type(key, struct frag4); - __type(value, __u8); -} slow_frags SEC(".maps"); +struct map_def slow_frags SEC("maps") = {BPF_MAP_TYPE_LRU_HASH, sizeof(struct frag4), sizeof(__u8), 1024}; // The IPv6 pseudo-header, which the ICMPv6 checksum covers and the ICMP one does not struct pseudo6 { @@ -155,8 +149,15 @@ static __always_inline int count(__u32 stat, int verdict) // into fragments gives up on the rest at the first failure. static __always_inline int to_slow(struct __sk_buff *skb) { - __u32 len = skb->len; - if (bpf_perf_event_output(skb, &slow, BPF_F_CURRENT_CPU | (__u64)len << 32, &len, sizeof(len))) + __u32 zero = 0, len = skb->len; + struct slow_frame *f = bpf_map_lookup_elem(&slow_buf, &zero); + if (!f || len < ETH_HLEN || len > SLOW_MAX) + return count(STAT_DROP_ERR, TC_ACT_SHOT); + // The same length, in a form whose bounds the verifier can see; the barrier keeps the + // compiler from folding it back + asm volatile("" : "+r"(len)); + len = ((len - 1) & (SLOW_MAX - 1)) + 1; + if (bpf_skb_load_bytes(skb, 0, f->b, len) || bpf_ringbuf_output(&slow, f->b, len, 0)) return count(STAT_DROP_ERR, TC_ACT_SHOT); return count(STAT_SLOW, TC_ACT_STOLEN); } diff --git a/bpf_integration_test.go b/bpf_integration_test.go new file mode 100644 index 0000000..87653f9 --- /dev/null +++ b/bpf_integration_test.go @@ -0,0 +1,54 @@ +//go:build linux && integration + +package main + +import ( + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +// A program whose read-only data cannot be written is refused before it is loaded. +func TestBPFLoadRefusesUnwritableData(t *testing.T) { + _, err := bpfLoad(bpfProgram{ + maps: []bpfMapSpec{{name: ".rodata", typ: unix.BPF_MAP_TYPE_ARRAY, keySize: 4, valueSize: 4, maxEntries: 1, flags: unix.BPF_F_RDONLY}}, + rodata: make([]byte, 4), + }) + if err == nil || !strings.Contains(err.Error(), "read-only data") { + t.Fatalf("want the data refused, got %v", err) + } +} + +// A ring buffer reader needs a map it can map, twice over past the first page; once its epoll +// set is gone, reading fails. +func TestRingReaderFailures(t *testing.T) { + for name, spec := range map[string]bpfMapSpec{ + "a hash map": {typ: unix.BPF_MAP_TYPE_HASH, keySize: 4, valueSize: 4, maxEntries: 1}, + "a one-page array": {typ: unix.BPF_MAP_TYPE_ARRAY, keySize: 4, valueSize: 8, maxEntries: 1, flags: unix.BPF_F_MMAPABLE}, + } { + m, err := bpfMapCreate(spec) + if err != nil { + t.Fatal(err) + } + if _, err := newRingReader(m); err == nil { + t.Errorf("%s: want the reader refused", name) + } + m.close() + } + m, err := bpfMapCreate(bpfMapSpec{typ: unix.BPF_MAP_TYPE_RINGBUF, maxEntries: 1 << 12}) + if err != nil { + t.Fatal(err) + } + defer m.close() + r, err := newRingReader(m) + if err != nil { + t.Fatal(err) + } + unix.Close(r.epfd) + if _, err := r.read(); err == nil { + t.Fatal("want reading without an epoll set to fail") + } + r.epfd = -1 + r.free() +} diff --git a/docs/recipes.ja.md b/docs/recipes.ja.md index 4ab7790..fbe2785 100644 --- a/docs/recipes.ja.md +++ b/docs/recipes.ja.md @@ -28,7 +28,7 @@ sudo sixup -wan eth0 -lan eth1 | [DNS サーバーを指定する](#dns-サーバーを指定する) | `-ra-dns self`、またはサーバーのアドレス | | [DHCPv6 でアドレスを配布する](#dhcpv6-でアドレスを配布する) | `-dhcp6s-mode stateful` | | [下流のルーター](#下流のルーター) | 上流のルーターに `-dhcp6s-mode stateless` | -| [IPv6 シングルスタックまたは IPv6-mostly の LAN](#ipv6-シングルスタックまたは-ipv6-mostly-の-lan) | `-nat64 jool` | +| [IPv6 シングルスタックまたは IPv6-mostly の LAN](#ipv6-シングルスタックまたは-ipv6-mostly-の-lan) | `-nat64 bpf` | | [LAN のホストの後ろにある別のネットワーク](#lan-のホストの後ろにある別のネットワーク) | `-ra-route`、`-source-filter=false` | | [コンテナで動かす](#コンテナで動かす) | `-no-sysctl`、`-state-dir` | @@ -274,13 +274,15 @@ sudo sixup -wan eth0 -lan eth1 LAN は IPv6 だけで運用できます。IPv4 アドレスも DHCPv4 も要らず、二つのプロトコルを並行して保守する必要もありません。クライアントは NAT64 を通じて引き続き IPv4 のインターネットに届き、その IPv6 パケットはルーターで IPv4 に変換されます。IPv6 シングルスタックへの移行は、ネットワークが IPv6 のためにできる最も意味のあることの一つです。そうしたネットワークが一つ増えるたびに、誰かが IPv4 を残し続ける理由が一つ減るからです。そうするつもりなら、この節はあなたのためのものです。 -Jool のモジュールをインストールし、NAT64 を有効にして sixup を起動します。モジュールが読み込まれていなければ、sixup が `/sbin/modprobe jool` で読み込みます。名前空間とは IPv4 の /31 `192.168.255.254/31` でつなぎます。これがホストのアドレスやルートと重なると sixup は起動しないので、`-jool-ipv4` で別のものを指定します。 +NAT64 を有効にして sixup を起動します。`-nat64 bpf` は sixup 自身の変換器で、何もインストールする必要がなく、Linux 6.6 以降だけが必要です。 ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool +sudo sixup -wan eth0 -lan eth1 -nat64 bpf ``` -Jool は veth `sixup-nat64` の向こうの専用のネットワーク名前空間で動き、`64:ff9b::/96` を LAN とルーター自身のために変換します。変換している間、RA はそのプレフィックスを広告します（PREF64、RFC 8781）。変換後の IPv4 は、ルーターの IPv4 ルートに従って出ていきます。MAP-E のトンネルを通るときはほかの通信と回線のポートセットを共有し、トンネルがなければ設定済みのほかの IPv4 の出口を使います。ファイアウォールは `sixup-nat64` を通る通信を許可する必要があります。 +ダミーデバイス `sixup-nat64` 上で `64:ff9b::/96` を LAN とルーター自身のためにカーネル内で変換します。IPv6 のクライアントはそれぞれ `198.18.0.0/15` から自分の IPv4 アドレスを割り当てられ、ルーターの IPv4 の送信元 NAT が、背後のほかのホストと同じようにそれを隠します。MAP-E のトンネルを通るときは、変換後の通信がほかの通信と回線のポートセットを共有し、トンネルがなければ設定済みのほかの IPv4 の出口を使います。このプールがホストのアドレスやルートと重なると sixup は起動しません。変換している間、RA はそのプレフィックスを広告します（PREF64、RFC 8781）。ファイアウォールは `sixup-nat64` を通る通信を許可する必要があります。 + +古いカーネルでは、代わりに `-nat64 jool` で Jool のカーネルモジュール（4.1 以降）を使います。同じ名前の veth の向こうの専用のネットワーク名前空間で動きます。モジュールが読み込まれていなければ、sixup が `/sbin/modprobe jool` で読み込みます。名前空間とは IPv4 の /31 `192.168.255.254/31` でつなぎます。これがホストのアドレスやルートと重なると sixup は起動しないので、`-jool-ipv4` で別のものを指定します。 クライアントが IPv4 に届く方法は二つあります。 @@ -301,13 +303,13 @@ RA の PREF64 に従って CLAT を有効にする組み込みの対応状況で NAT64 を通してプライベートな IPv4 アドレスに届くには、ネットワーク固有のプレフィックスを使います。Well-Known Prefix ではプライベートアドレスを扱えない場合があるからです（RFC 6052）。 ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool -nat64-prefix fd00:64::/96 +sudo sixup -wan eth0 -lan eth1 -nat64 bpf -nat64-prefix fd00:64::/96 ``` クライアントに CLAT を有効にさせず、DNS64 だけで NAT64 を提供するには、プレフィックスを RA に載せないようにします。 ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool -ra-pref64 off +sudo sixup -wan eth0 -lan eth1 -nat64 bpf -ra-pref64 off ``` ## LAN のホストの後ろにある別のネットワーク diff --git a/docs/recipes.md b/docs/recipes.md index f8f95de..608a591 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -30,7 +30,7 @@ both. | [Choosing the DNS servers](#choosing-the-dns-servers) | `-ra-dns self`, or the servers' addresses | | [Addresses handed out by DHCPv6](#addresses-handed-out-by-dhcpv6) | `-dhcp6s-mode stateful` | | [A router behind this one](#a-router-behind-this-one) | `-dhcp6s-mode stateless` on the upstream router | -| [An IPv6-only or IPv6-mostly LAN](#an-ipv6-only-or-ipv6-mostly-lan) | `-nat64 jool` | +| [An IPv6-only or IPv6-mostly LAN](#an-ipv6-only-or-ipv6-mostly-lan) | `-nat64 bpf` | | [Another network behind a LAN host](#another-network-behind-a-lan-host) | `-ra-route`, `-source-filter=false` | | [Running in a container](#running-in-a-container) | `-no-sysctl`, `-state-dir` | @@ -398,19 +398,26 @@ packets at the router. Going IPv6-only is one of the most useful things a networ IPv6, since every such network is one less reason for anyone to keep IPv4 around; if that is your plan, this section is for you. -Install the Jool module and start sixup with NAT64. If the module is not loaded, sixup loads it with `/sbin/modprobe jool`. -The namespace is reached through the IPv4 /31 `192.168.255.254/31`; when that overlaps an address -or a route of the host, sixup refuses to start, and `-jool-ipv4` gives another. +Start sixup with NAT64. `-nat64 bpf` is sixup's own translator: it needs nothing installed, only +Linux 6.6 or later. ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool +sudo sixup -wan eth0 -lan eth1 -nat64 bpf ``` -Jool runs in a network namespace of its own behind the veth `sixup-nat64` and translates -`64:ff9b::/96`, for the LAN and for the router itself. While it translates, the RA announces the -prefix (PREF64, RFC 8781). The translated IPv4 leaves by the router's IPv4 route. Through a MAP-E -tunnel it shares the line's port set with everything else, and without a tunnel it takes -whatever IPv4 uplink is configured. A firewall has to let traffic through `sixup-nat64`. +It translates `64:ff9b::/96` on the dummy device `sixup-nat64`, for the LAN and for the router +itself, in the kernel. Each IPv6 client gets an IPv4 address of its own from `198.18.0.0/15`, and +the router's IPv4 source NAT hides it like any other host behind it: through a MAP-E tunnel the +translated traffic shares the line's port set with everything else, and without a tunnel it +takes whatever IPv4 uplink is configured. When that pool overlaps an address or a route of the +host, sixup refuses to start. While it translates, the RA announces the prefix (PREF64, +RFC 8781). A firewall has to let traffic through `sixup-nat64`. + +On an older kernel, `-nat64 jool` runs the Jool kernel module (4.1 or later) instead, in a +network namespace of its own behind a veth of the same name. If the module is not loaded, sixup +loads it with `/sbin/modprobe jool`. The namespace is reached through the IPv4 /31 +`192.168.255.254/31`; when that overlaps an address or a route of the host, sixup refuses to +start, and `-jool-ipv4` gives another. Clients reach IPv4 in one of two ways. @@ -439,14 +446,14 @@ To reach private IPv4 addresses through NAT64, which the well-known prefix may n (RFC 6052), use a network-specific prefix. ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool -nat64-prefix fd00:64::/96 +sudo sixup -wan eth0 -lan eth1 -nat64 bpf -nat64-prefix fd00:64::/96 ``` To offer NAT64 through DNS64 only, without clients turning on their CLAT, keep the prefix out of the RA. ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool -ra-pref64 off +sudo sixup -wan eth0 -lan eth1 -nat64 bpf -ra-pref64 off ``` ## Another network behind a LAN host diff --git a/docs/recipes.zh.md b/docs/recipes.zh.md index d1a2336..a12b420 100644 --- a/docs/recipes.zh.md +++ b/docs/recipes.zh.md @@ -26,7 +26,7 @@ sudo sixup -wan eth0 -lan eth1 | [指定 DNS 服务器](#指定-dns-服务器) | `-ra-dns self`，或服务器的地址 | | [用 DHCPv6 分配地址](#用-dhcpv6-分配地址) | `-dhcp6s-mode stateful` | | [下级路由器](#下级路由器) | 上级路由器加 `-dhcp6s-mode stateless` | -| [IPv6 单栈或 IPv6-mostly 内网](#ipv6-单栈或-ipv6-mostly-内网) | `-nat64 jool` | +| [IPv6 单栈或 IPv6-mostly 内网](#ipv6-单栈或-ipv6-mostly-内网) | `-nat64 bpf` | | [内网主机后面的另一个网络](#内网主机后面的另一个网络) | `-ra-route`、`-source-filter=false` | | [在容器里运行](#在容器里运行) | `-no-sysctl`、`-state-dir` | @@ -272,13 +272,15 @@ sudo sixup -wan eth0 -lan eth1 内网可以只用 IPv6，没有 IPv4 地址，没有 DHCPv4，也不用同时维护两套协议。客户端仍然可以通过 NAT64 访问 IPv4 互联网，由路由器把它们的 IPv6 包转换成 IPv4。改用 IPv6 单栈，是一个网络能为 IPv6 做的最有意义的事情之一，因为每多一个这样的网络，大家继续保留 IPv4 的理由就少一个。如果你打算这样做，这一节就是为你写的。 -安装 Jool 模块，再启用 NAT64。如果模块没有加载，sixup 会用 `/sbin/modprobe jool` 加载它。与命名空间之间通过 IPv4 /31 `192.168.255.254/31` 相连；它与本机的地址或路由重叠时，sixup 拒绝启动，用 `-jool-ipv4` 指定另一个。 +启动 sixup 时启用 NAT64。`-nat64 bpf` 是 sixup 自带的转换器，不需要安装任何东西，只要求 Linux 6.6 及以上。 ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool +sudo sixup -wan eth0 -lan eth1 -nat64 bpf ``` -Jool 运行在独立的网络命名空间里，经 veth `sixup-nat64` 相连，转换 `64:ff9b::/96`，内网和路由器自身都能使用。转换期间，RA 会通告这个前缀（PREF64，RFC 8781）。转换出的 IPv4 按路由器的 IPv4 路由出去。走 MAP-E 隧道时，它和其他流量共用线路的端口集；没有隧道时，从已配置的其他 IPv4 出口出去。防火墙需要放行经过 `sixup-nat64` 的流量。 +它在 dummy 设备 `sixup-nat64` 上转换 `64:ff9b::/96`，内网和路由器自身都能使用，转换在内核中完成。每个 IPv6 客户端从 `198.18.0.0/15` 分到一个自己的 IPv4 地址，再由路由器的 IPv4 源地址转换隐藏，和路由器后面的其他主机一样。走 MAP-E 隧道时，转换后的流量和其他流量共用线路的端口集；没有隧道时，从已配置的其他 IPv4 出口出去。这个地址池与本机的地址或路由重叠时，sixup 拒绝启动。转换期间，RA 会通告这个前缀（PREF64，RFC 8781）。防火墙需要放行经过 `sixup-nat64` 的流量。 + +内核较旧时，可以改用 `-nat64 jool`，运行 Jool 内核模块（4.1 及以后），它在独立的网络命名空间里，经同名的 veth 相连。如果模块没有加载，sixup 会用 `/sbin/modprobe jool` 加载它。与命名空间之间通过 IPv4 /31 `192.168.255.254/31` 相连；它与本机的地址或路由重叠时，sixup 拒绝启动，用 `-jool-ipv4` 指定另一个。 客户端访问 IPv4 有两种方式。 @@ -299,13 +301,13 @@ Jool 运行在独立的网络命名空间里，经 veth `sixup-nat64` 相连， 要通过 NAT64 访问私有 IPv4 地址，需要用网络专用前缀，因为知名前缀可能不能用于私有地址（RFC 6052）。 ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool -nat64-prefix fd00:64::/96 +sudo sixup -wan eth0 -lan eth1 -nat64 bpf -nat64-prefix fd00:64::/96 ``` 只想通过 DNS64 提供 NAT64，不让客户端启用 CLAT，就不在 RA 中通告前缀。 ```sh -sudo sixup -wan eth0 -lan eth1 -nat64 jool -ra-pref64 off +sudo sixup -wan eth0 -lan eth1 -nat64 bpf -ra-pref64 off ``` ## 内网主机后面的另一个网络 diff --git a/docs/standards.md b/docs/standards.md index baf2548..792a7ef 100644 --- a/docs/standards.md +++ b/docs/standards.md @@ -455,7 +455,11 @@ Tested by `TestNATPMP` and `TestPCPServerAgainstKernel`. ports are kept inside the line's port set by a port restricted NAT, as a MAP-E CE must. - **RFC 6052, NAT64 prefixes.** The well-known `64:ff9b::/96` by default, or a network specific prefix with `-nat64-prefix`. -- **RFC 6146, stateful NAT64.** Done by Jool with `-nat64 jool`. +- **RFC 7915, IP/ICMP translation, and RFC 6146, stateful NAT64.** `-nat64 bpf` translates + statelessly in BPF, with an IPv4 address from a private pool for each IPv6 source, and leaves + the state to conntrack and the IPv4 source NAT, which together make a stateful NAT64. Fragmented + ICMP and IPv4 UDP without a checksum, which need the whole datagram, are reassembled and + translated by sixup itself. `-nat64 jool` uses Jool's stateful NAT64 instead. - **RFC 8781, PREF64 in the RA.** Advertised while the NAT64 translates, with a lifetime of 3 × MaxRtrAdvInterval, and withdrawn with lifetime 0. - **RFC 6877, 464XLAT.** The CLAT is on the clients; PREF64 is what lets them find the prefix. diff --git a/flags.go b/flags.go index 1c8020b..926a5d5 100644 --- a/flags.go +++ b/flags.go @@ -54,9 +54,9 @@ var ( tunMTU = flag.Int("tunnel-mtu", 0, "MTU of the tunnel device, 0 means the WAN interface MTU minus 40") tunMetric4 = flag.Uint("tunnel-route4-metric", 4096, "metric of the IPv4 default route added through the tunnel device once its IPv4 is known; the metric is high on purpose, so an existing IPv4 default route keeps winning and the tunnel only takes over when there is none. 0 adds no route") tunCap = flag.Bool("tunnel-capture", true, "when running as a daemon and DHCPv6 sends no tunnel option, capture tunnel traffic to infer the parameters") - nat64 = flag.String("nat64", "off", "which NAT64 implementation to configure: jool uses the Jool kernel module (https://jool.mx, 4.1 or later; when it is not loaded, sixup runs /sbin/modprobe jool once), run in a network namespace of its own behind the veth sixup-nat64, translating -nat64-prefix for the LAN and this router alike. The namespace exists, and the prefix is advertised in the RA unless -ra-pref64 is off, only while the module is loaded, which is checked every minute. The IPv4 output leaves by this host's IPv4 route, through the tunnel's source NAT or any other, so a MAP-E line's ports have one allocator. A firewall has to let traffic through sixup-nat64: the LAN's, forwarded, and the answers to this router's own, arriving on it. bpf is an experimental translator of sixup's own that needs no module but Linux 6.6 or later: it numbers IPv6 sources from 198.18.0.0/15 and leaves the rest to the source NAT. off configures none. DNS64 is not part of this and is left to a resolver of your choosing") + nat64 = flag.String("nat64", "off", "which NAT64 to run, translating -nat64-prefix for the LAN and this router behind the device sixup-nat64, which a firewall has to let traffic through: bpf, recommended, is sixup's own translator and needs only Linux 6.6 or later; it numbers IPv6 sources from 198.18.0.0/15, which must be unused on this host, and leaves the rest to the IPv4 source NAT, so a MAP-E line's ports have one allocator. jool uses the Jool kernel module (https://jool.mx, 4.1 or later, loaded with /sbin/modprobe jool when it is not) in a network namespace of its own, for older kernels; it runs, and the prefix is advertised, only while the module is loaded. off runs none. DNS64 is left to a resolver of your choosing") nat64Prefix = flag.String("nat64-prefix", nat64WKP.String(), "prefix the NAT64 translates (RFC 6052), advertised in the RA while it does (see -ra-pref64). Reaching private IPv4 addresses needs a network-specific prefix such as fd00:64::/96, and one carved from the delegated prefix would change with it") - joolIPv4 = flag.String("jool-ipv4", "192.168.255.254/31", "the IPv4 /31 between this namespace and Jool's: the lower address is the gateway on this side, the upper one Jool's pool4. It must not overlap any address or route of this host") + joolIPv4 = flag.String("jool-ipv4", "192.168.255.254/31", "with -nat64 jool, the IPv4 /31 between this namespace and Jool's: the lower address is the gateway on this side, the upper one Jool's pool4. It must not overlap any address or route of this host") unsolicited = flag.String("unsolicited", "request", "unsolicited IPv6 from the WAN to the LAN (RFC 6092): request lets in only what the LAN started, endpoints that recently sent out, ports opened with PCP (UDP 5351), and the ICMPv6 and IPsec that must pass; deny does the same without PCP; allow lets it all in. PCP, and NAT-PMP for programs that know only it, also forward ports of the tunnel's IPv4 while sixup does its NAT. The RFC 7084 border for ULA and LAN prefixes holds in every mode") srcFilter = flag.Bool("source-filter", true, "let the LAN send to the WAN only from its own and the delegated prefixes, answering others with ICMPv6 code 5 (BCP 38), and the WAN reach only those; turn it off when another prefix is routed into the LAN") tunNAT = flag.String("tunnel-nat", "auto", "maintain the nftables table sixup for traffic leaving the tunnel device: auto configures the port-restricted source NAT a MAP-E customer edge is required to have (RFC 7597), an ordinary source NAT on a line with its own public IPv4, none on DS-Lite where the AFTR translates, and in every case an MSS clamp to the tunnel MTU; off writes no rules. Nothing outside that table is read or changed, and it is removed on exit") diff --git a/go.mod b/go.mod index 4419ab3..75fcc4a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( ) require ( - github.com/cilium/ebpf v0.22.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/nftables v0.3.0 // indirect github.com/josharian/native v1.1.0 // indirect diff --git a/go.sum b/go.sum index c778bbc..fbf178e 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,5 @@ -github.com/cilium/ebpf v0.22.0 h1:v2ktp0roffpMOj2MMf3idtCQZOsAoC4BJbAJN+ke2bY= -github.com/cilium/ebpf v0.22.0/go.mod h1:CDzZbe2hC5JjlDC+CY3KFCzlYwN4gbxppYM+Z10bQt4= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= @@ -24,12 +21,10 @@ github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA= github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= diff --git a/jool.go b/jool.go index 5be6e8b..833f4d7 100644 --- a/jool.go +++ b/jool.go @@ -691,11 +691,12 @@ func linkRequest(typ netlink.HeaderType, flags netlink.HeaderFlags, hdr, attrs [ return err } -// checkJoolIPv4 refuses a /31 that an address or a route of this host already covers: the two -// hosts behind it would become unreachable, or Jool's traffic would go to them. The default route -// covers everything and does not count, and neither does the veth of an earlier run, which the -// kernel may still be tearing down when sixup starts again. -func checkJoolIPv4(link netip.Prefix) error { +// checkIPv4Unused refuses a prefix that an address or a route of this host already covers: Jool's +// /31 or the BPF translator's pool, whose addresses would become unreachable, or whose traffic +// would go to them. The default route covers everything and does not count, and neither does +// sixup-nat64 of an earlier run, which the kernel may still be tearing down when sixup starts +// again. +func checkIPv4Unused(link netip.Prefix) error { ifaces, err := net.Interfaces() if err != nil { return err diff --git a/jool_integration_test.go b/jool_integration_test.go index 12847a9..fd4d982 100644 --- a/jool_integration_test.go +++ b/jool_integration_test.go @@ -30,7 +30,7 @@ func TestJoolNamespaceAgainstKernel(t *testing.T) { enterNetNS(t) loUp(t) link := netip.MustParsePrefix("192.168.255.254/31") - if err := checkJoolIPv4(link); err != nil { + if err := checkIPv4Unused(link); err != nil { t.Fatalf("an empty namespace has nothing to overlap: %v", err) } m := &joolManager{prefix: nat64WKP, link: link} @@ -92,7 +92,7 @@ func TestJoolNamespaceAgainstKernel(t *testing.T) { // A second run may start while the kernel still tears down the first one's namespace: its veth // is no conflict, and building a new pair replaces it - if err := checkJoolIPv4(link); err != nil { + if err := checkIPv4Unused(link); err != nil { t.Fatalf("the veth of an earlier run must not count as a conflict: %v", err) } srv.Close() @@ -114,7 +114,7 @@ func TestJoolNamespaceAgainstKernel(t *testing.T) { if err := routeSet(1, netip.MustParsePrefix("192.168.255.0/24"), netip.Addr{}, 0, 0); err != nil { t.Fatal(err) } - if err := checkJoolIPv4(link); err == nil { + if err := checkIPv4Unused(link); err == nil { t.Fatal("a /31 inside another route must be refused") } routeDel(1, netip.MustParsePrefix("192.168.255.0/24"), netip.Addr{}, 0) @@ -218,7 +218,7 @@ func TestJoolStartWithoutTheModule(t *testing.T) { joolSkipIfLoaded(t) enterNetNS(t) loUp(t) - if err := checkJoolIPv4(netip.MustParsePrefix("127.0.0.0/31")); err == nil || !strings.Contains(err.Error(), "overlaps the address") { + if err := checkIPv4Unused(netip.MustParsePrefix("127.0.0.0/31")); err == nil || !strings.Contains(err.Error(), "overlaps the address") { t.Fatalf("a /31 inside lo's 127.0.0.0/8 must be refused, got %v", err) } m := &joolManager{prefix: nat64WKP, link: netip.MustParsePrefix("192.168.255.254/31")} @@ -263,7 +263,7 @@ func TestJoolStartWithoutTheModule(t *testing.T) { t.Fatalf("with room for %d descriptors, want EMFILE, got %v", extra, err) } } - if err := joolWithFDs(t, 0, func() error { return checkJoolIPv4(m.link) }); !errors.Is(err, unix.EMFILE) { + if err := joolWithFDs(t, 0, func() error { return checkIPv4Unused(m.link) }); !errors.Is(err, unix.EMFILE) { t.Fatalf("listing the interfaces needs a socket, got %v", err) } if err := joolWithFDs(t, 0, func() error { _, err := routes4(); return err }); !errors.Is(err, unix.EMFILE) { diff --git a/jool_stub.go b/jool_stub.go index 6c0d2be..1c9cb0c 100644 --- a/jool_stub.go +++ b/jool_stub.go @@ -17,4 +17,4 @@ type joolManager struct { func (m *joolManager) run(ctx context.Context) { <-ctx.Done() } -func checkJoolIPv4(netip.Prefix) error { return nil } +func checkIPv4Unused(netip.Prefix) error { return nil } diff --git a/main.go b/main.go index 69175ad..6e3b0e0 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,10 @@ var licenseText string //go:embed NOTICE var noticeText string +// bpfPool is the private IPv4 pool -nat64 bpf numbers IPv6 sources from. The source NAT on the +// way out hides these addresses like those of any other IPv4 host behind the router. +var bpfPool = netip.MustParsePrefix("198.18.0.0/15") + // nat64WKP is the well-known prefix of RFC 6052, the default of -nat64-prefix. Carving one out of // the delegated prefix would tie the translator to a prefix that changes, and every client would // have to be told again. @@ -161,10 +165,15 @@ func main() { } // Refused before anything is configured, so a conflict leaves the host as it was if *nat64 == "jool" && !dryRun { - if err := checkJoolIPv4(joolLink); err != nil { + if err := checkIPv4Unused(joolLink); err != nil { fatalf("-jool-ipv4: %v", err) } } + if *nat64 == "bpf" && !dryRun { + if err := checkIPv4Unused(bpfPool); err != nil { + fatalf("-nat64 bpf numbers IPv6 sources from %s, but %v", bpfPool, err) + } + } layout := shared64Layout(*shared64) switch layout { case "wan", "lan", "split": diff --git a/nat64_bpf.go b/nat64_bpf.go new file mode 100644 index 0000000..e04989b --- /dev/null +++ b/nat64_bpf.go @@ -0,0 +1,38 @@ +// Code generated by tools/bpfgen from bpf/nat64.c; DO NOT EDIT. + +//go:build linux + +package main + +const nat64License = "Dual MIT/GPL" + +// The maps, the read-only data last +const ( + nat64MapStats = 0 + nat64MapSlowBuf = 1 + nat64MapSlow = 2 + nat64MapSlowFrags = 3 + nat64MapV6Of = 4 + nat64MapV4Of = 5 + nat64MapRodata = 6 +) + +var nat64Maps = []bpfMapSpec{ + nat64MapStats: {name: "stats", typ: 6, keySize: 4, valueSize: 8, maxEntries: 8, flags: 0x0}, + nat64MapSlowBuf: {name: "slow_buf", typ: 6, keySize: 4, valueSize: 16384, maxEntries: 1, flags: 0x0}, + nat64MapSlow: {name: "slow", typ: 27, keySize: 0, valueSize: 0, maxEntries: 4194304, flags: 0x0}, + nat64MapSlowFrags: {name: "slow_frags", typ: 9, keySize: 12, valueSize: 1, maxEntries: 1024, flags: 0x0}, + nat64MapV6Of: {name: "v6_of", typ: 1, keySize: 4, valueSize: 16, maxEntries: 65536, flags: 0x0}, + nat64MapV4Of: {name: "v4_of", typ: 1, keySize: 16, valueSize: 4, maxEntries: 65536, flags: 0x0}, + nat64MapRodata: {name: ".rodata", typ: 2, keySize: 4, valueSize: 48, maxEntries: 1, flags: 0x80}, +} + +// Where each variable sits in the read-only data, and its size +var ( + nat64VarSlowMark = bpfVar{off: 0, size: 4} + nat64VarPoolBase = bpfVar{off: 4, size: 4} + nat64VarPoolMask = bpfVar{off: 8, size: 4} + nat64VarPref6 = bpfVar{off: 12, size: 16} + nat64VarMask6 = bpfVar{off: 28, size: 16} + nat64VarV4pos = bpfVar{off: 44, size: 4} +) diff --git a/nat64_bpfeb.bin b/nat64_bpfeb.bin new file mode 100644 index 0000000000000000000000000000000000000000..5a3ec4b84b919133a5370f03ff07e202441a490e GIT binary patch literal 20376 zcmd5^e{kGKb>BOk>z> z35+o_y-=Kn45g+t0Rjd8ij&Y3LV8oBMV-?y(@v5}8R+l_?T|LK&>G?y)J}gmiS2Jc zpU=KMomVGWC$(wskMH}weedmiyZiR-+x@OoS>y5!zuqFB)4$3a{QItJ+!DUk5|8g* z?)fXr+lKfi zp37?`;p?s5)hnI*X_nMft6RGJXj&y+@8i?CrHSgP_IkR?_%QhlfA_QS|q$`!l&{7n6#L;n!z7Ec`F9Z|ze#5JyaSEpC99Jpri$b;!?rukTY{tWSn1AIC+bU62G z02?Z+J^#sq{4pQJyK22h>>=*oP%8<~Psh8Q{I4nA?IcGOk9WrXXYz>0>AT2}c=wBh zA^yhvSV#0bktn{WuzQdN6BZA{_{VXta<#@(un{A}l9-x+$zn^z%WpKimF# z;^z?)`9jE8)a-e1?9zIUTBUJ$D)O-%6G@`KQLe@Ze5yAb~X7|wMD(v*k79btCRd_kJh{0$GpE^*q*CWlxrTLU5t;*;0GV0 z9QHZUP6qk#wD6ZlhHg{Z}TW?b~3L~zcsb9&3SbG6!mwy z^FJWW4f}ag26XP`lsJBw$PQP(Q{}E+^l;*B9tOL%Bc)#KzS560Hk$iTo_P*m15`X_a#;tFy(Q%<4ss_Xk)%`e%f&(&yt$)b7E!oxu;cJD;anJC}qw z6v@vukABtkR|qFw%_sb>p!jea^$AD(YM!r*XRArTdbyQw^jNBO0r;DVPguUQ$H$G_ z=k7)1XM^vxR?n-9PwNQi%7a1w8KbK|fU7@2 zSDp&o=Qme4<(a^L&iIu_0)HLx+4^k<`~ko2`t5>lDf%1x@udD%k9TX-G49JfKIZv* zN39%os6QL^%q>@Fl2gCWQ!d{mr}1gXU2k?;J~(Rr8P+MR>np*Lsei!pRUY(yALc_@ z?Mc&F^5Cgx-Mo5c9_)UFHQPMRwLeeKO;NtnojXl<@F8!X)knX)o8_1vRl<;WK+ngB zhjq%v<+44)EOi|NiaMBY!qXS`J)C}&gr+#Gd24z$K#%qfxx*U&f8Y10j|+7h_gX^f zd!7sP{18#+u0rQ({5zdn!Gfs2^0gvMaZXec2{Lq@q3`RaZ=BHT>1cnvJLIXIu0!+I*K^Nvh`W=lH(Ced{IM(KkK=>nZk%sjvYm%) z_(k%)(Reh@)na*!2dZeaYa!$D8OmckK22D^k543#Rrym9e*164(O(5Z>p!kHDnGE= z=Lz*Sq&l}h5hxx!nx5iY-RAdx?@ycGTU7j8+EeuTGa&u2qtj(?Ab&aG)1~iC+@Aq< zoJ`!00JA+wmT?%c!LJ&FJ+M#n(BqjL{95)|zgMB>9gxohZUMa1^mak7;=SoTHNmeYh7>HuZDA?b;S(#S(&n@XC|!OSf`rw=eXGXBkM8y=x=S#lk7iH9`^kM;k@%s zt(J%{Z=+Ph&sM`pmJ*Nimbz9qjMJ?90{0h`vvw@Rzu52eFIiL@CgyHvif5l>LFE6l zo@2!RZ}r`@(we#mJLP|{b0h7sdg%8}{X2dAu=fP{%7^(()7_7z zNBDNvAfM1%7UX*MS-!Qn zWqu--ANKVJ|3IAiF-vMyKJopbhY)zeXUla>`Pst#;Ou{}=1zC^UkUd;vgr7}iuFYO zFz@)j9XVbL`BNA6gKM1g#q$I5n)ea^=8zB6H7l{sUpNkz;ymDYsdo|m_$=kok53V< zOEMn@=&lus;e5^ff8s5|UAaW~>gR__PI+@#d?(#d(P>^<+(%r`C`0n&pLP3=`FU3^ zP(J1P6V{X5X>WJ#H1|#TtqAiN^Hw#(zw#su6|%~oBEKE*^}GCme#`4hs9moi}hqMpA|n%Pa{r-_=3FQN!khcxWPvZ9x45JS5fzN&M`ke8JcmfoC(`E~HP|E{w8m_L6-T+x1ZTzoEn@R+CX?Mt7bbZ$>cAcy?m<9ze>TyTye|IHP(UY_1} z+<))0{_KaHQJ!z&@!)dYGSM2{CtBh%bAJ!aD|RzWF===OD+^CrF9xcoOsCdn~VPOP?nk z4SurGx2IlaDfV-|UHQ@L{k@o|ynu3z-2Rf-dr^=4GmMv&=TWZyjmirs*LdwuOy2B= zU7CMGB}YHBedATB#r?+l+?Imm@;%C5Tpq?qKjS*`=<3Qu_2&)0d-V0<$NJ*qlgY0* zXy1*yKIQ4*JgBnD;=y&c4@pE%Zd>E|5YO)98~CIb`w{)_N(7qu6rU@-#<$+{RW@7w zTYddC-{1W>&;1BFn)V|fB7RqiPwYqDBIpjDNQe2>Lf`vX9*z6XkS~~@Ia|kP>_@D8 zb7i&ptMA{yUyAa+$w*Jt>R&sLeac5=XAhsj|7Y$`wmb7b<2%sj$FElOa^ocr4|%}z zwrlxn>pSzmv&5sga(1@$b)|dxJ>>6nFDoQiJzQr`v)tpk4AyPUPit?!48OzgY)|3b z4*2`fA=+Ck5jz%_UK%0h~?3E4n_Ij zT<3CHuOs<(_HB1lf0N(IzoYtQJLf2k4^p~uesc-@K=OzSA7uH+hkgDrG_3FX{WcEF zA16tl<@@R6Y<^O&QfjX6YDl6k{I0J!5c%O8dGX6yhcVt-*O8}bzJ$0x=KZb1{40aJ z<_zsIzYFV$3+1*>TvjfUD{%uB5App<@fQ2LkCB4@AZVPQ+jq1xD3``{!_~SVm+Pir5_T8bB?mC(f17GFRATc-9qYr z&g%bL!f1Swc>Iy?PLa>z4Zovn-c)1{?R58&{(pG>cq8?je=lJF`_k7bkNzGfoag-8 z9M8Y)3qD7=z+V;+zh6?m4Ew)C`>h`sC)->H{Q8XX)ck0wr_Fin^Qr^pA3W#7uW=u@ z(&O(J!`@L6OwR(&6Xw>VBob~3J@2C)%%g;GE$sQ?jR(Glyl^e?$86rR{Ll-}>_hqb zxcnPWeC2wSZ-~pU`_2omn@|*YZmdeG>we7ecpl+iwSI~l{#DP))gHM+TCtF>@i_9} zD&L~{Ipbac_McPa8!z$c!hP*5{88&b^u1=LyyfNf&61x{59Yh}Ll?8d;zLK=Zsaer zFT{_gb>yF^5qYA1k)1{Cn?)%nB2VF7@2mMXRurBK`F|}DUdBG6#5atS{1@kbw;E2s zb3o-0m=^=YO^)T;h(q61gs5k$;BkL10`(P$=Mp~MiqVJteL$=;r&s{_TL@?Dhvi+^ z=Ov_KpJ)BsZF&F58&vLWo$4J~>-Pr@{+*uQ{caCOt~LHdurOfzuCR~Z;rpe#9^`Bv zKXSdvZ?XMbN%%CM?SIUU{K#89|G*{>!#Tx(<>9z@F-S_!aE742a=Y(`vP#o*?>0XU zT~<%!cE7#`dQ-W^Dd2l=_x&F5iFdJjjJN7_g-~8Dury7(-b@I3B2ehkC}94Wof9im z?#9IL3gr^`v$8Px1T-absdVYi;;prq}H7rSN<6r*Ln!YdYVllILRf%oay*pLajy=V@2=6|`$l z8+LWPf_BMs=h-p_Le>^l|}=We4s;@mr?>&f$; zh*qp9@7~#L7x?Gehie^)icXm?nr>b~AkAHNss`^zHM zs~%d2_9}C3u|MTGeEfGO@qN3-_vk~c%;~$D&e6@T1)LXMWIu4H^knOrMYR9S4_VT5 ze@T91`WJJ5iS6&`DJpNy0J6WbfJ~ThkI6j&81jWno9>G!{{$cN+#Fy6c=u4zJqB1c zyLj&V8sI^`ao;4|<180Q#UGIr+(Ur5yAqQv=P{4^b4SRpCT{v@pX{i4KAcCYUmM>EU%S~BF%E0-ES`<3Gs_61$x+$mR7K6vJ2QT`RqHw)JOQ|OluL9E;C>@eA|m$HZ$@KJ$%xB;L|fF{MyH@=c>=M z{NTgscMX2a;8O-C2~Sov?>0y4qVRAUH1-04>s?7W;h8b92s*(#c`9FV0V%jN1Df=5I}Z7n^wxYg)L!o&O2t@g402LbUI3!Z1(v%TAX7 z{~pUt56^K$F&z9pZwz^6IDcGMjlNU#vKA~r2_CJ#Q(R8E)tjx|Nlf1p+MGw3;}ksL z?bkX0v~k`$op1fuIQioJRMY=II81stzbF@p&Gml|G6@-){_nx3h`0E=bLBd_mS?n5 zwa}kcJ$)(jlKJQt|E^6LwHW$4B;TKYTJl%$*~tI-e-}iDe&!vm+Mi=N&L97Pa31X! zKlGEG?e7%qPwyve(SFf)Eq5=G-lF}=h@t-h$#3Kn_Rrq`;@`E&O_2}!HD7T)`x)Xb z^otJtT9;br*SZD$pOyT@^@|_+pC>=`YhMcepOgH|{udqk|Bmz)`ZX@l|DfbAu3!An z|1I)Ezv343e_rx4^@|Su&ye0iKl4+L{}%}7G5+F*{z>vfzwRSJ{~r=>(f^`D|DTiI zLcihw^nX$E7jM7#q5pC6L%-G==>L-BXSQE-=>Ib5(Ykshwr6@>b$?BM#P@p%%LP8& z6zu#sU&azXU2Z_Tj7A(QBY%8~1f1hPO;}6#4EqiIeS1~=u*UcBDDOBN=;6F|WShUY zhx=R_Z`Av3R+t;+)2#@9M^P>j#rb!i*Wb04IP$V>K3)gB>$Sf8z&5)lcp{x-3I2{t z{(K44@GEdM_Z^_8o#fvFtEep6IS1tucy#;oi865!|H|KC*VRp64r5(-(Y;3 zqkTa&^3%y$N%CuF+YhiF*!zQ4^`rbW%1^O|urCSy;^RL2IPAAU!y^#b*1 z-9$e6B)K*#2PuUe|?rPjAO`m8E@gYXF~puJo+g4to^jRN<3O$s&lL_@>e8BTPSDs@ZMXS z`R}yv+C#*ndE3tYvQ;hQ7rPIfZ1`uw+FjTOt{oo~9?psRcc+YR(fNU! z$3AezcY=2NQ*i_Rsrb?~zHQdi@;=K#{6zWtZ2OyVpJhBz|AqS(tLk?Ochmcy|I~Q? EH_g)Y*Z=?k literal 0 HcmV?d00001 diff --git a/nat64_bpfeb.go b/nat64_bpfeb.go index 643f900..87d8077 100644 --- a/nat64_bpfeb.go +++ b/nat64_bpfeb.go @@ -1,192 +1,155 @@ -// Code generated by bpf2go; DO NOT EDIT. -//go:build mips || mips64 || ppc64 || s390x +// Code generated by tools/bpfgen from bpf/nat64.c; DO NOT EDIT. -package main - -import ( - "bytes" - _ "embed" - "fmt" - "io" - "structs" - - "github.com/cilium/ebpf" -) - -type nat64Frag4 struct { - _ structs.HostLayout - Saddr uint32 - Daddr uint32 - Id uint16 - Pad uint16 -} +//go:build linux && (mips || mips64 || ppc64 || s390x) -type nat64In6Addr struct { - _ structs.HostLayout - In6U struct { - _ structs.HostLayout - U6Addr8 [16]uint8 - } -} - -// Names of all BPF objects in the ELF. -// -// Used for safe lookups in a Collection or CollectionSpec. -const ( - nat64MapSlow = "slow" - nat64MapSlowFrags = "slow_frags" - nat64MapStats = "stats" - nat64MapV4Of = "v4_of" - nat64MapV6Of = "v6_of" - nat64ProgNat64 = "nat64" - nat64VarMask6 = "mask6" - nat64VarPoolBase = "pool_base" - nat64VarPoolMask = "pool_mask" - nat64VarPref6 = "pref6" - nat64VarSlowMark = "slow_mark" - nat64VarV4pos = "v4pos" -) - -// loadNat64 returns the embedded CollectionSpec for nat64. -func loadNat64() (*ebpf.CollectionSpec, error) { - reader := bytes.NewReader(_Nat64Bytes) - spec, err := ebpf.LoadCollectionSpecFromReader(reader) - if err != nil { - return nil, fmt.Errorf("can't load nat64: %w", err) - } - - return spec, err -} - -// loadNat64Objects loads nat64 and converts it into a struct. -// -// The following types are suitable as obj argument: -// -// *nat64Objects -// *nat64Programs -// *nat64Maps -// -// See ebpf.CollectionSpec.LoadAndAssign documentation for details. -func loadNat64Objects(obj any, opts *ebpf.CollectionOptions) error { - spec, err := loadNat64() - if err != nil { - return err - } - - return spec.LoadAndAssign(obj, opts) -} - -// nat64Specs contains maps and programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type nat64Specs struct { - nat64ProgramSpecs - nat64MapSpecs - nat64VariableSpecs -} - -// nat64ProgramSpecs contains programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type nat64ProgramSpecs struct { - Nat64 *ebpf.ProgramSpec `ebpf:"nat64"` -} - -// nat64MapSpecs contains maps before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type nat64MapSpecs struct { - Slow *ebpf.MapSpec `ebpf:"slow"` - SlowFrags *ebpf.MapSpec `ebpf:"slow_frags"` - Stats *ebpf.MapSpec `ebpf:"stats"` - V4Of *ebpf.MapSpec `ebpf:"v4_of"` - V6Of *ebpf.MapSpec `ebpf:"v6_of"` -} - -// nat64VariableSpecs contains global variables before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type nat64VariableSpecs struct { - Mask6 *ebpf.VariableSpec `ebpf:"mask6"` - PoolBase *ebpf.VariableSpec `ebpf:"pool_base"` - PoolMask *ebpf.VariableSpec `ebpf:"pool_mask"` - Pref6 *ebpf.VariableSpec `ebpf:"pref6"` - SlowMark *ebpf.VariableSpec `ebpf:"slow_mark"` - V4pos *ebpf.VariableSpec `ebpf:"v4pos"` -} - -// nat64Objects contains all objects after they have been loaded into the kernel. -// -// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. -type nat64Objects struct { - nat64Programs - nat64Maps - nat64Variables -} - -func (o *nat64Objects) Close() error { - return _Nat64Close( - &o.nat64Programs, - &o.nat64Maps, - ) -} - -// nat64Maps contains all maps after they have been loaded into the kernel. -// -// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. -type nat64Maps struct { - Slow *ebpf.Map `ebpf:"slow"` - SlowFrags *ebpf.Map `ebpf:"slow_frags"` - Stats *ebpf.Map `ebpf:"stats"` - V4Of *ebpf.Map `ebpf:"v4_of"` - V6Of *ebpf.Map `ebpf:"v6_of"` -} - -func (m *nat64Maps) Close() error { - return _Nat64Close( - m.Slow, - m.SlowFrags, - m.Stats, - m.V4Of, - m.V6Of, - ) -} - -// nat64Variables contains all global variables after they have been loaded into the kernel. -// -// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. -type nat64Variables struct { - Mask6 *ebpf.Variable `ebpf:"mask6"` - PoolBase *ebpf.Variable `ebpf:"pool_base"` - PoolMask *ebpf.Variable `ebpf:"pool_mask"` - Pref6 *ebpf.Variable `ebpf:"pref6"` - SlowMark *ebpf.Variable `ebpf:"slow_mark"` - V4pos *ebpf.Variable `ebpf:"v4pos"` -} - -// nat64Programs contains all programs after they have been loaded into the kernel. -// -// It can be passed to loadNat64Objects or ebpf.CollectionSpec.LoadAndAssign. -type nat64Programs struct { - Nat64 *ebpf.Program `ebpf:"nat64"` -} - -func (p *nat64Programs) Close() error { - return _Nat64Close( - p.Nat64, - ) -} +package main -func _Nat64Close(closers ...io.Closer) error { - for _, closer := range closers { - if err := closer.Close(); err != nil { - return err - } - } - return nil +import _ "embed" + +//go:embed nat64_bpfeb.bin +var nat64Code []byte + +var nat64Rodata = []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0} + +var nat64Relocs = []bpfReloc{ + {insn: 1, m: nat64MapRodata, off: 0}, + {insn: 26, m: nat64MapStats, off: -1}, + {insn: 46, m: nat64MapStats, off: -1}, + {insn: 51, m: nat64MapRodata, off: 28}, + {insn: 58, m: nat64MapRodata, off: 12}, + {insn: 68, m: nat64MapRodata, off: 28}, + {insn: 74, m: nat64MapRodata, off: 12}, + {insn: 84, m: nat64MapRodata, off: 28}, + {insn: 90, m: nat64MapRodata, off: 12}, + {insn: 100, m: nat64MapRodata, off: 28}, + {insn: 106, m: nat64MapRodata, off: 12}, + {insn: 116, m: nat64MapRodata, off: 28}, + {insn: 122, m: nat64MapRodata, off: 12}, + {insn: 132, m: nat64MapRodata, off: 28}, + {insn: 138, m: nat64MapRodata, off: 12}, + {insn: 148, m: nat64MapRodata, off: 28}, + {insn: 154, m: nat64MapRodata, off: 12}, + {insn: 164, m: nat64MapRodata, off: 28}, + {insn: 170, m: nat64MapRodata, off: 12}, + {insn: 204, m: nat64MapStats, off: -1}, + {insn: 210, m: nat64MapRodata, off: 8}, + {insn: 217, m: nat64MapRodata, off: 4}, + {insn: 228, m: nat64MapStats, off: -1}, + {insn: 243, m: nat64MapStats, off: -1}, + {insn: 254, m: nat64MapStats, off: -1}, + {insn: 269, m: nat64MapSlowBuf, off: -1}, + {insn: 280, m: nat64MapStats, off: -1}, + {insn: 309, m: nat64MapStats, off: -1}, + {insn: 324, m: nat64MapSlow, off: -1}, + {insn: 335, m: nat64MapStats, off: -1}, + {insn: 342, m: nat64MapSlowFrags, off: -1}, + {insn: 351, m: nat64MapSlowBuf, off: -1}, + {insn: 362, m: nat64MapStats, off: -1}, + {insn: 377, m: nat64MapSlow, off: -1}, + {insn: 388, m: nat64MapStats, off: -1}, + {insn: 403, m: nat64MapSlowFrags, off: -1}, + {insn: 412, m: nat64MapSlowBuf, off: -1}, + {insn: 424, m: nat64MapStats, off: -1}, + {insn: 432, m: nat64MapV6Of, off: -1}, + {insn: 440, m: nat64MapStats, off: -1}, + {insn: 470, m: nat64MapRodata, off: 12}, + {insn: 509, m: nat64MapRodata, off: 44}, + {insn: 557, m: nat64MapStats, off: -1}, + {insn: 596, m: nat64MapStats, off: -1}, + {insn: 610, m: nat64MapStats, off: -1}, + {insn: 619, m: nat64MapStats, off: -1}, + {insn: 646, m: nat64MapStats, off: -1}, + {insn: 697, m: nat64MapStats, off: -1}, + {insn: 711, m: nat64MapSlow, off: -1}, + {insn: 722, m: nat64MapStats, off: -1}, + {insn: 743, m: nat64MapStats, off: -1}, + {insn: 808, m: nat64MapStats, off: -1}, + {insn: 849, m: nat64MapStats, off: -1}, + {insn: 898, m: nat64MapStats, off: -1}, + {insn: 911, m: nat64MapStats, off: -1}, + {insn: 944, m: nat64MapStats, off: -1}, + {insn: 966, m: nat64MapStats, off: -1}, + {insn: 982, m: nat64MapV6Of, off: -1}, + {insn: 990, m: nat64MapStats, off: -1}, + {insn: 1036, m: nat64MapRodata, off: 12}, + {insn: 1073, m: nat64MapRodata, off: 44}, + {insn: 1218, m: nat64MapStats, off: -1}, + {insn: 1237, m: nat64MapStats, off: -1}, + {insn: 1246, m: nat64MapStats, off: -1}, + {insn: 1287, m: nat64MapSlowBuf, off: -1}, + {insn: 1299, m: nat64MapStats, off: -1}, + {insn: 1332, m: nat64MapStats, off: -1}, + {insn: 1344, m: nat64MapStats, off: -1}, + {insn: 1351, m: nat64MapV4Of, off: -1}, + {insn: 1367, m: nat64MapSlow, off: -1}, + {insn: 1378, m: nat64MapStats, off: -1}, + {insn: 1402, m: nat64MapStats, off: -1}, + {insn: 1411, m: nat64MapStats, off: -1}, + {insn: 1420, m: nat64MapStats, off: -1}, + {insn: 1433, m: nat64MapStats, off: -1}, + {insn: 1456, m: nat64MapRodata, off: 4}, + {insn: 1459, m: nat64MapRodata, off: 8}, + {insn: 1467, m: nat64MapV6Of, off: -1}, + {insn: 1475, m: nat64MapV4Of, off: -1}, + {insn: 1484, m: nat64MapV6Of, off: -1}, + {insn: 1487, m: nat64MapV4Of, off: -1}, + {insn: 1518, m: nat64MapRodata, off: 44}, + {insn: 1586, m: nat64MapStats, off: -1}, + {insn: 1591, m: nat64MapRodata, off: 4}, + {insn: 1596, m: nat64MapRodata, off: 8}, + {insn: 1604, m: nat64MapV6Of, off: -1}, + {insn: 1610, m: nat64MapRodata, off: 4}, + {insn: 1615, m: nat64MapRodata, off: 8}, + {insn: 1623, m: nat64MapV6Of, off: -1}, + {insn: 1629, m: nat64MapRodata, off: 4}, + {insn: 1634, m: nat64MapRodata, off: 8}, + {insn: 1642, m: nat64MapV6Of, off: -1}, + {insn: 1648, m: nat64MapRodata, off: 4}, + {insn: 1653, m: nat64MapRodata, off: 8}, + {insn: 1661, m: nat64MapV6Of, off: -1}, + {insn: 1667, m: nat64MapRodata, off: 4}, + {insn: 1672, m: nat64MapRodata, off: 8}, + {insn: 1680, m: nat64MapV6Of, off: -1}, + {insn: 1686, m: nat64MapRodata, off: 4}, + {insn: 1691, m: nat64MapRodata, off: 8}, + {insn: 1699, m: nat64MapV6Of, off: -1}, + {insn: 1705, m: nat64MapRodata, off: 4}, + {insn: 1709, m: nat64MapRodata, off: 8}, + {insn: 1717, m: nat64MapV6Of, off: -1}, + {insn: 1727, m: nat64MapStats, off: -1}, + {insn: 1743, m: nat64MapStats, off: -1}, + {insn: 1770, m: nat64MapStats, off: -1}, + {insn: 1800, m: nat64MapStats, off: -1}, + {insn: 1819, m: nat64MapStats, off: -1}, + {insn: 1994, m: nat64MapStats, off: -1}, + {insn: 2017, m: nat64MapStats, off: -1}, + {insn: 2026, m: nat64MapStats, off: -1}, + {insn: 2067, m: nat64MapStats, off: -1}, + {insn: 2095, m: nat64MapStats, off: -1}, + {insn: 2122, m: nat64MapStats, off: -1}, + {insn: 2131, m: nat64MapRodata, off: 28}, + {insn: 2137, m: nat64MapRodata, off: 12}, + {insn: 2147, m: nat64MapRodata, off: 28}, + {insn: 2153, m: nat64MapRodata, off: 12}, + {insn: 2163, m: nat64MapRodata, off: 28}, + {insn: 2169, m: nat64MapRodata, off: 12}, + {insn: 2179, m: nat64MapRodata, off: 28}, + {insn: 2185, m: nat64MapRodata, off: 12}, + {insn: 2195, m: nat64MapRodata, off: 28}, + {insn: 2201, m: nat64MapRodata, off: 12}, + {insn: 2211, m: nat64MapRodata, off: 28}, + {insn: 2217, m: nat64MapRodata, off: 12}, + {insn: 2227, m: nat64MapRodata, off: 28}, + {insn: 2233, m: nat64MapRodata, off: 12}, + {insn: 2243, m: nat64MapRodata, off: 28}, + {insn: 2249, m: nat64MapRodata, off: 12}, + {insn: 2261, m: nat64MapV4Of, off: -1}, + {insn: 2299, m: nat64MapRodata, off: 44}, + {insn: 2424, m: nat64MapStats, off: -1}, + {insn: 2433, m: nat64MapStats, off: -1}, + {insn: 2465, m: nat64MapStats, off: -1}, + {insn: 2474, m: nat64MapStats, off: -1}, + {insn: 2516, m: nat64MapStats, off: -1}, + {insn: 2525, m: nat64MapStats, off: -1}, + {insn: 2534, m: nat64MapStats, off: -1}, } - -// Do not access this directly. -// -//go:embed nat64_bpfeb.o -var _Nat64Bytes []byte diff --git a/nat64_bpfeb.o b/nat64_bpfeb.o deleted file mode 100644 index 4c7b60f12b16a5191ec3921250d4a7527c4ad101..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 70672 zcmch=3v^u9b*6dnE<7j{Un2OBbt#cl0RqV)2tp8Fx)fiaM3EJVG!feh3Iss{Ly#x} zAPJgwTu$3=w&R{u+)j*moOIceEjZ(8SKRK9y~eXplVmbX;&@eOsaJE>IGvhao(W=W zrfXuS8;&K{{QI1JF3#NqD3bEHmI~*7cki>$KJUl9MSbkx(L1&R$-8CzfdW`Pe)K*p9 zz?aHt;+Z!3t5lt*?W9|bT;HQV9KQnbtB9|$yv~Fb*SVyue>JN7x1=Zh^B3`Nh2!5e z#d~=F%Kw?-BmEyzDD!b?8{JjvQd>DrVP6ORMS3QvJf-ida*Jnf*uR&aH&?HAmcK(+ zmzJ8gacO5+TkKnIC&_2VAtYkj2uZ5?l^sRyL5 zTi62W>v}@^e@OZZq+hM=w3Rz4ypQy?KGOfXj<*y=j1SV^LE!@FYdO;YD#;f}U-t*( zze~sGdbmgawLa4SBI%?2>vXuY{EUu&clm2M(*JFeFOa_OuSmaJ$A90NB(zHcpvF&eWX94<1JN7bhxv;N5{WA|5}dp&yswB^dHgj+sb=&{C6*Zt&j9S zq~k4B%{ts!eon`~JAExj`tK(>k0CmKGsh&S+(zWIG zFzL3EMWjphsfNE$)+7DKjLy&U_mCgmeqeWvw)^4oAqr=nqQAD>o-3bBs}^7d%5N3j z@9(0&N)5bL*7bkR%ts#82j+Lvcp*Q|{bl9_(#E{)Yk9A2XR`io>eivExxK8$JidwI zIqfY?EZSRIwjZax>G`Ne{p@EH@A{Wb`w!>u@iOk~{)K#AfZ9^Di%*a~+MRBvwwin0=pC&o_;{gi6Hz-6suclD+tGvxGw~rY2t4j8h);rx4&&lTl_N$-$pA;YY{68E! zRo3tOFR!YCu~YRO3hzIjI!O9XyP)=>9A)eeibuvyXH7AEj_c_fm{UQ9T~L zW#?P;V?DoKT2t;JJ=ELd6mq)M&-FNo^f%CbgdT@_z5&OGG4Jf7aE8Y}?g!Q5ua$Mm zc0M}nk(R0*|8iOPm+@&cpZ9xuK2ZZ-HuDTvf536N==W3;gEuKJo=#`!+{(B&D}xp%y% z6Hk}>ea4Ff@wS`#;eAN+jPXY`bAHCz&lz1C7?0AlFIN>aDebqq_60LV+kS8=O83iU zo}VI{*Dts}EZOmy^`5xtYkSW6Q0eiL`%Cph^iy5G7$_A_MFqFbB+g`f8P1O zOuFyCq;)%pqrLq-vE6rZIy4{X`)E%klF#VT z6z$L>QgP_d(uLi(Ts7^cdoZqOoT^(HS3b!RQv0c@QI(GppYrIhDi3<b>3yX23+?DF(lcC9jdrQyarvi_UI)Yxgfy-;a{5h3UytLQ{yQhV zN0I-=h0j*?w*sdwbE2K1p`i&xh%c6BHlqNLyq(v`%o!Lsem%(d`1|@_RZx zpZ=>v{dpnEC&>o}Kl+<_1Zir|S!H{juwByAQkmni)U) zW2~oF3y9Za?DhAWbl~snWZ+bW{+7!FUn^(n>Rf{UD)0CS9jfbJDNoVW0_o_OvOa8i z-1JO*ZjUihAM0lHDl1r@m4uyHjvwLp5j#)o%2R4xFwgON8~yYLl;X`a{l$6oqZEQY z`n%wG{$9t)?PMA4K(9N_Prv8!QO^^i9PWF3JgLk&NW`BFmxBtim~YzdIGByXEr6D2=PtZyCR<-{R*7D{MbTzkP}{ zG5$m-JQ=Oj>lV>&kJ@&6ZKqksaC`McIj!UJiSqsU%CsKWmoi70Ygwd6|7IF9_ENS{zHswZ~t!;zec_N?;Sg0T&tjQzUm(G3iQJT$2azi=GR*kA7PHe zX%4d->hT70J-#5+6PwDoiN+->EKmzyw9o>gsghI4K6e-RXja~)LZ}t7rWhCYJ zbiNo>`VQ;S`Qn56`@3Wh{{9YyTrSm{pDgQs$6P1p|CRA_CDbG6Uq3-u&w903zv~7s z%InPs*xx^>|GMhn|HsLHtk=Fvp3#kUx)^RN|2c)%cA9jWc^s>Y z>-FYa=U+3=Z@4^n!k?1X=lW-VgZ3~lH+t|-uZQ~Myl#M9e7t_2*A1PT71*p75{Bjc z-r}44_U|RFjr?cLdH_u8R$^MG(t04H!`IBZ;8G`_E4)_zGTjI3@d5ks*O;|MuwHLK zPVHTRzru3e{=i>l*7Fef4dVWtkJ+mIa z!Cz-K=L%mb{}0TxURRL+E6oS^T;cVy?%yq3ek7yMC7lb@zrkkF53=?gq1iVIGrCl; zZ!p+0&i7$Qmv3O_d84pMyr>gTx99te7YX8>&Ys+ZZ-2b_V2Hize$S1k6ZmO zl#BZQ3Hpn2H2z0$&h$Ov@g3%TY9=g?;=XBboo0P=)hBJ$GTv$Wg{>#PAHjWHuHt?a z_f0?OFy+VjX&L8!eeDHaC;0k&i}y>9mwDV*_I%-RRHq+Vsn;JW0_Sx9pm81NQR$Ut zzZYTt`1m%hkNL%nBOHH)eGb_Cn?`QWG5S~Wcwo};uwT&a{zj_zF&vx`s?Ks zT`iE_5ZxE`-pliyZD)$-@w@gDe7_szG)s24e67AO)@^x_PMX{ChWprO{F?UjQTl7! z|K0nGlb8!gZr35`H*;D)PJa>4*MVI}(>amuiJV_rzr9TwVqAH9vH5kS`l%kDYt&Em zJkNIM9Q`e_W4O8k^Y&k8_UNyQoHFg??Um7=Q9o%yd!_atKKM?Oc$&ZRGyb;-$$K1bZ?JP*o~-0sU;JUxDxO$hGWaaBB*4bJOG z-~F&3)t@vs>A9>=mt0>Dv7h+qaf*+6f0070*G#|R@$9N;cU4?J6?Q$w?Jwx>_CDux ziJ2!nb)4C`WG+IBd)TB|Ev9+f7~G%)t@ zcnN>>I)n8e;<^4$w@?Zwhc*iBdDbcIjXjSqzlD9wdre1=+(_p!N=J+vo*XyqdXnp# z-ly9`@%l}@(4;v=%=vzZ^E@+8USoT_zn1UkVW0kwKcaL*|6Q!Vbj$v7@}bWyLgoDs zEoz;WBmrzmgTj%~Yqi1AUcFc&ND{zuLGo$&XM$zSvrkHV)=Kfm|P zwLic*Y6I~%xZTtJPyX=F&*Oef-v6U-{;M6h-!AVz_2v(^nsF#iBK?{9xC#4o@5j0= zt-1Ldd|$Rpyoahka(wE4!rbrE-r#^pw|2eGOp`>k_q*N~!9J`+dN+Og%ZFAtUq}1- zFx{BZvlALq+N{H*Uv}L)(@olU+d==)`;B+=gZo?6V)ku+oJs3s)XKE_0bOAora!{Z zIQFYra>~=<65fNvW$a_M8FdVuzS1pmVYV6r%kdrI6#%{TGOl z-s2R)PLB>>Gta@tzh>T(vhSbOc_dXUFwV@<1>!$X;a&Sp-RgpUbh9mAyI;PfPg#R` zuI17O-YQFVmY{E+$O?b}JVbq?V+kL8;Kb&~4aJ(HH9^n0vuX)}+?<;s7 zid;Hq^pAI%(C!!PI(Oi8^FG_k43=D&Z?1j7l*6e6>6gpHFPM60W;@qDVCF3=ckV|h zhUMW8m~yu~^GhTR_Cry7-_9@1`;460+x%NwnBTbnT=ld(+N^5_>FhW9Gh8oL@03sX z%VnuQv%=_Kg?)UY&CGA;^vY$N2kLecUXS7Z@wkciNfXfX)!(K|9{*@OP7~i}f1>&8 z{~>)8w(c*of7x+mhQ|+XpA{|a2gRk2eNW!Lf7s0H47>-ICi%Phv)DL;_aetB{(bsY z`3v+b`2c=Z{Q~`Z=>hzz{ssEA^8x&-`33r=ck=hyzG{Dge(4cmF~4ZO#i#BUHQ%Ej zR#JjI&eS~kIP;TlQT!V9lN^O7Q~Gx??RI=K>!0!*$>?LpxtqR5yU(NjzPsLknDH(_ z7BSwLc{)lIq4yQ^j40OAz1BaT&+oM#*7J>s?~C~MJ&hG|J^pjZ*AC8?9(nD#mzmF0 z#mQ*ZBP1989fe+r?sNV;F7Jr&Zyn1!R%Y}%qL$g7C&l@G@LVp)cbu#vU0(;b9`W3* zQm^;l!@RYMETSHlQ@HT{P~GqQTEX*Oo>%EPmo6{%?;*}_-en(`B(GKFOsx*7Upz}! z&igdFJ#qT)<$anun61KlEVRz9K=>%aN`w`3ALaL_2yMS^bl$(vD;em~dtcQEHzNG_ z$+*fROtW8f4*Mk7#t+`#u;;d?bpOK3`BC+6p;mp)BW+c6%7ooCzExB-kT3M9`WWw{ z;K$$54cOQ9d(PPLu5xIeLo0jE*=yQkRW;VjvlOomdN>z;OU5&L#a zo2S3Ld&=kw=$x{LRFc%A>Q=kaT_ zzHNuSH&ceW{9|%{ljE&@zmv<4-jhE<7BJ45aiY!a3o%~!bibMV4W1+EeW=5SguF+v z-e)-adR}z+n9xrM);mb6Z{~5*Fdy5l?7rYslyMAAV9N7kd9mkFV&*df#(1O$ree%v+veTTdLH_FJta;d*(9^@pi1{H|y> z*%^4FT%_>&8|HZ}!UFSc4)YXpdfl5__Y$}@;n}AziB?x>9HM} z&mQo+&Af+he`n>`Nt1u?w!dRP=yz@&>BG)C3V#mof3=Vuv97uAb=OJky%&ft-~S5U z=NA369`SWMGVKR`JWAGradm@jKlgYK?DJY*d+{*8=l-^!F4x@6)7-Z)WJP zukf7nyL2`DVgCK=8uK26eI9x;s@DbW_8jVzN7^`NH-7nezt&xMM>eloupQnK`M=2R z(0LEO;xt)y+Rp>F!=K%u_{i^@6e7Pl3eld66kav{df@L7b2=6FIR-PHcbIkWwNIPp ziyPD8`G#&)ICt=1WZ~xx~xL4TNX&4))jQV`B@#mgZ^Uuw0Qbj z|JBY4UBXDeiu7^*`Y$M4Abl-I`pZbZK>F2mAL)ON!u!a-)<^of-y;1Q9e-!}w{(1- zk4~BTpnnHI%aOkJdx7+8b^JE-JN5fWU+cfxspfRN`QO3oabSV`)B0V#x=iW*y+HaV zA8qEnqWef+>m&VHPG9$Hl>hG#FOa^LBmMtG@&(dwp!+EQFHm?N>1%zYe@Dlg{~i2) zr{mvU{`&a<(*G-xFOa@o$0GeN>iF;8{QzfJn6|0O!yS^ixe|L*j)9O?fN$rngp zj{`_wzps8D`PceL|CUa7{&(>Ia*_1)`$I_onr(Vrt+4arDW4GM`)wxuX1)8wxcGt@r!6$j_y97_rZFj&uB(P9s0wZ9<&4 z>w4$(y{o>yZ^Ni9m*$w!$9s_aW8V{J{Xoy&H~B4>ZG5Do`)>>4yU$1L^PIl@#;#pQ z++~;hX|NAqenZ>C^N-ice?=CsE}`+S0_|AO=PS+oUuoQ@as1=lKJ9y9X=Z9?X8!k} zg$0m3599dltk*5}6KS!I%-{5N{0)okBghWc-``)zKAk_{{w!Ir&s}FYpJskUd$;>H z9(RU!^8Rg}&HinkbqllmCg{JQ?a#~2{*4*$*GEW>^3>y(eV&MZc1*`xMt{+7+RSq| zyN?6EO!qDS|Guw#lv1;>&J0~9R9^(ClcW#iM z_}zu>@oeWI_EYbCjP1}l39V!Ge6ZL$7XPlrH%ZUF&wn*agx|k;I>dWG!SCnwI)(2C z|E|NijMiIX-ZR8|Kzg-@@#fQJ{fze-zC{|?4}XWkQ`&yHY}d&&Uemdd-A7>kt=9`U zFG-WWn3vM`n3r_>#iHr}#pCj$_wC+Zz|KA1+k58}>7yQAq0rdT=lJg@7XA1^-fL@F zApU#ZPv(Pm>5+A1_mdlLCN$gj`E%G$hW2R>)4touBa~~5Le%dA*2{wL<rIl}-8hFM6Cacklo2fy@0P-b+VLY(1RN zO~K6|j32&e2`6If^f) z6!U>|6rJv0#+U0v{VJ~4;Y*(z_|o$f`h1qlU=^-E&&g1rZ##ZGihEQZWcMo^XAQ2u z&ev;k{Y}SBnb*~jQ+!hYpE-^WoHQO|9n!OPPWLDx-ShCJy3l(Ogj5$x@%8bgx}eoY z8egglde143?`^IN57*kg`?N*z)A*7tvYW^EyKL(TTx(lg9~;37wPo^|9H%T&J&^Ak z@TJe2_|mnH?_cwE8sRLbu?g318hXd!locgS_UrMrW%V>&->pMxfet-a`oD39Zc@Ad z5x%;N5&nd)J%nX^sm^@3L1|H4C-AjtQF|aGWXHpo@)pCFK2$Gl`1<%#z0g>m#+S-+ z3%+^StmS-e1-tp|04{~mCs>c#6*rd(suQ~QphIcVwU2Ac6I~~9y^F8YxIVzwSzP}T zU*~a6evp0<*MsJoT&TFn*B-8^zd)zL$M;iwoy7HT;Ceg4G`^oT_bWYI)42)sD}7vl znXi+$euJ;mxX$r)7T34H5HB(CXx8rM{JsOy?6 zuDfv!J2iQPZaagnZU-G53$HSNEl|CN6PtgTFG%m&UmA>D$7j|ly_i#wQ?sKl6cdifdHS(ekJJ(0%b=nTW zW(Bj)A(id1ItE;SD7YTJIHCIb_PzY_zF+Qp<;VvP_MQ0U#KFEk+`GKJLtVZ&K78)t z;2F=TUGD0fzcn&?@tjA0)Wo&apt{gM**AD`Vr+PDT%8}hrmpmlTpm>86a5q8YWQMT zU;mjiWBTHl|@>{&H3 zcu}R!PxNu_#?Grtmj^Eo_Fd>trG}B~)Y$06=)mZRx-!y#u`e|?IF9Tam>3wgccBB# z;nA_-iEGIIIjD{I4WAvpcxG@?*>GZ@&)gU|t%mx?hZJrMjP#F>51&zI`X~DJAC&GH zb+JD++=sho`!5WSTvKC%7e*%r`-W4UYGibve?(seH@kwHT~;$iWoA_MC0XygFo+~3 zP*oRF>gw>=U>{}u-1unU_#jF|Ur=!x4O1RNS6Tlo6&*@^T#cW{f2ZN$1X{_&fI6Kz zyH&%M0aZVI){8;9dGCe(vGd-ZJ)SOdf>!Uks;_rsUVImBj15j)9=qtlNZ;7t88|gC zVGMAYwR_Je4)q=CJ96ys%LiY1rFED3sFRAV8%l{x3{8w)9FHA5(bJdcJCb;RS8JDq3P!r`LATGuF`} zU}5z9+(>a?CQmd(V=n5$Cgz8HJxk7$VKXQh=}G2O_E<6 zoSYasGsfPtWRG_wk$Cy|iQ|2TUfy^3rGv*#nCc^+D3RT^q|OetdLQ|SF8h0xulKZ# zKt+oZJ3AC)9VTow#i*Xh^%1zfMR(m+WcpWAqoX6}kK<@Wlx?EZ{o{jN(oh{98X?IJ zQ)g}lx7a2U${=)Y22boyc$+qHkvR8H9$5JPk^L_vOfo3&iO~sB;9GEu`ju@Cq!Ep#ct6<7&AGx~t2!}QM4i-Yi9j{)pD z8UmUeeaLIFemR8^eb5d=A?rv17EERBJNC=en#N5z+UePA)8&Ljthvyrre{}J3A!NH z(|Xe(=gp05R}WowAFcdyq;qVTvlKNP~%!3(Dc&veFUK(|fAHWn%iH|mwn z)abYwShwwX6{(yCaBlA!37wvz8@nCRMn}%va}4aCpc)xD6SNzOKwW}sg}Nq&gCQy$ zDlFZ0sbXj{8wrL)Jwyy6rYTV#E68V79gp2L0HAQUql>OzA4pz0qz%|1fk*ZEX#n=@ z$Y4i)Dl;leXCeuuN`(;&?=+Zm0g_)dCtzuJ@!TL5PMCAdtUK?qGgG&FaXSsz!OB|b zKl9&Q9-rtN8y&r14Dd9K#JHIsc%kp*J9h8{+c9fNZ*XF0aEw-VA(x$Gg|p)m zqgWR?D+G+k+4w`9IQ^XZA_VQnm{ddpk-6@N_o4XO2gK?C0p?$v6SwL^ENx- zZ)jM)tBgiEQPkmO1DB1HfL>D0Tis4pd4)+{qZx1B%L^W4#I|(m+3)Wn-Fjz!fvTBd z^nE)zX%iZ#A9ABejSpTvGuq{)Y$vYYLcL;ou*XU?09Xhn^N=Zq!hLGxGpzigd*K#gxUjUYVIL4)U!4kP5>*2wAFWMY!;qI&3-MhMfUj6QVJ09vaD77}ge9yf<3-d!D$KHg34J?{6 z=b<1lIMUgtYkzDoHPSyY7~YZygFq~;&^V~pZNqmuWruFQ)bXxhy-9nYzL$<1>wD?M zNxgDds1IJz^&jdVv{Ia3C*7l&IZf0~uI*lTsOh2*?fgrtLfdj_M|E}tTW~Zuv$adS za`5DV<9$5`UZ&;bUGpw2#Ghl;m-fBVMq zx_8^<*@ERh(upFr64UyOFVqrjv1`Yjx%<*(Y|Fbm>gv~NrAaA-7YFaMG?=gD(JjLR z7g9)U6RuK0%ORlUr|r;NMmqK4!Kfo$ba76YRD+!m?aH_;olVU{ZFbOrLc2ztjxKDR zig4O`wBH^K{aDO*Q@i7u>DuLO-RdRyAZ4d_40}$mXR!aw;8?H>nqQobj}0JSG*{fU zjg2-UxY;S2_M&USEOp4VJqFPwZx2LF5zpt#=HFcz>&S~P*>!o- zq8+({HKb#kAri8yq5EFAlM!7i7cmOxayor^cw}PuB2M-$45Y5bXpJ-qnKOFOZ~;}_ zMS@)bA0W)1dpMhKUDsI8!I*9|=*cve1f*B#aKp5y;C45Sv>n}O6x%wUr36v-=f*~9 zHOfO*&=K^mvGx9Y2uGxn>oPqB_Vun~k4}P8qlMjKU6hty&{=G4=bRtFh+Wz(vaO;dObJE-7bSLG6ruOL=>-obS+0Mz8GYtnEHG zHTPay?R@ur(fs@uEUtnke$i=3)GzeHZGO)c47b$VsBrJvSlG?iy%V7Y4B8M6jo7Z+ z5rbNmE+BKJDt8qW31isZlaezQ=rXsbcsT9D0KVNG-qkNSU|1#0@%nUdn?6X{w!Kvk zkp~WW10#d|V>|T#nHnA(z}R=8|C)Dt&>Ogn!Sh`I@I`Xl?m^74xt+>JV(`rlTGY=r z2Ynb*m2D3?yYF#nfdhKex+s;sbRdM!A_D_7?Y8I&51w}~59sL-S_YN^?B*3S#JAxv z6Or1k@NL8lpAc;_iwPuR+q7K@KJ4Ngm5wbOZ3ySjsojECX?-)Ei(BE}>QN<2c9OyXR(LcC-$D)ecS+V1pGo6E{JI+FpmuHkX zJV3MCwnMi?F6e$PXLAr09J|vZ2I^0H!*n=5e>Sh9+Hs8PCUi(?mu~&Ke$23R?D)an zBd_2joBPmuYJzvi3~CjNV??K2Ll9FRU)X48@K`1|6%7*9e%)uAF%#t#a!9YG%rlbU z9Fk5eu}Pt=jk~hqQ^{b4GpC{`z3XJhu77l>Pw#Lp3L#ku)^_H;d3N9w`?%gt1j~On z1#=QeCzjBkZ(|<}A8MkO-(_^7BPBd@x@wL-%>h%40=-5=ht#W9xX8bl@>m@qYA~ybAGu-AF))}qg1%_ECEIRMq z*9iqVZU=&UP8w*D&Snicwp;W2-8+Z&$cyc&&EDB|t@|9A%)Zeozw3_Hne(^Nuw=$B z+ihrzGeT#EbS4;dbUte6v_}mX6FT%Xd4=YT4t*L&USA0Ycj{P+pGAs@(iOfQIr9I8|j#g}$cf)`# z%*Boi@3Gu;+KN0R3=R@?yBSVlY0O)1bWyvBp>;lIaZE2Z%uGqz3-xBxnRQXw-uNKB znFlU?UebB*^O9iaOgovM`|$}qprDGn?_uD6#{p*;z#>(T188JO>AuF1(Bc!3f3By8 zG=SfAc&NAE3ob741o^-rYL;f*I-H`(iYnbX=WzB~tU3SCU2l5l&j56rp#@&>7<^RM zD%qXCrB6Zb@g_n4#C#DUez=h=xa;Lw4)2g@hoFzxoF@r-*I;sCj@{4b$LW;g#a5X5 zjE-MW;7+t2v5=%LRyu{@0~xt=(1jz{4ltqP(Q6ENi@~DPYkA)h?!NEu5{EMKNXD!) zbdteB!y4z4Dy~|y1E4BdWUaxuG`p0=UL=4+baFri%4Y|3B!F5C-FcQepw6#d8DUf( z>mQiFl3e?0*5cMty(5R4Z75)SZ=n$iM?^eG1*wJ8r6USDprDx`G(FoBIjZsb$1%_Bvn zHIzwu(FGF?l?!#E&_Pqshj`Gyt~5Mm$nCDK{{@Gi#rL)tOzr-A)H$<5nQ`}7Y(J~D z?T(fj`TUNSL{#eh2FLaLUiU3F-aq7UAG9!PNaC5XUBkWmR&=3}K6nF9VZt6$N0013 zcP>Ci{pceI*J~{^b3JR z9te-&5Fmbpwe$D@ZI-qd<%S!-c|0!wS9MWC$90HE8VN6_tNpH{2;#PRMQE( z7p+L&GW}-2bC}+bjWd5rHTB~CS9-r$>wjJ~eTd%CPB8z4YMP`pk&c%CrD{4u_LB~i zzZvF#RFM<(&hE6}g5jzis`4nMH_JSzDihGJV1KGEsD?S>9+q29h2vM;5b-&E{J(W8 zDgBBmmj8jO{Ba97=Wqmk%VE4zUQuv(BiQ;!@t**f968yWa~SsMt#B%zD$<9&%BaJz zSBW@*9QGM5DTiUND(W!oRZ$#UzOYy2ISg4dnJd--kig*N3X>P z{WXXLss4GFxm4RCVL5o$=;B|WH03~*_(8j?4=zhdl`qx-ju^+ zFY7SbyX7$1%R5Z=3J#OKS%=A9$zifL=P>NmD2HLMCh9Qk)gX>rel?!MkkxcMO!j<- z$zH-?vNz-~{HsYhO!g)nCVOdz$zH}`vNz>0*}KL3H!3oM-`dsOHe4N1ks-_nH6>>J z-(+a!SA8Fn$MAJL_;O*yiUysKy9>YEwJMa$?LkbxLr|u$_YFq*T>awiQ1O#)RbwaS z?+E9w_HkADk=3`dWS%oN^dzkH`&&(Rd;`!FLP?{+F1o|8vaje|-zb z|GaA4jrLg2{?~p>HTq}|^>LOHuLpNKO!9=o2f)1!lRU}%wrV_p#!x?LxUNw(9z}lZ zrv%&j;`*$=W8}6z>x<0ne*APP5m^a969Qz;fBMgpN6c%sGo-0f^B`Uzl{+ixBfOpnK^%r-ON^>%RA6#|84#n z+5fuFs>T!OkBvhnzGkG=ILZ8)YV1XOZ%i})8`bzB($6vfv1&}h|Hgu0TON&fnA4^_ z8jFq`<Z2p?qKbya%tchRu4qQk7ZMtPR@|d(nEh;GRyp3EY09A{p^qO z7a_*@^GL*Dl5>3o@)jYtgm3IrdEgviI5%Pq?BBriBj@;eTb zyy!5=XB{Sa$zhVuISe_Pl&=5!dR0X~BW-SR811v!V|{LK&2b@5IE?i{bFaf>FX=GZ zyWueDXB;N|DThfv?=b0e`2_wJ969N8{Noo43oB6^GM{R`EY$NUz(%{;9dzvF?e|5kp7<$tLf zZ*MjKQ~~wRdX_JZ7&(4o-1sKOucfd-@`kml@eV3;X^eS~YW&`l;JCx6|D`_jMb-Fy zir*`Eh#CI9h5A^UaTx27rJNs||E27|&HqyN-;TdaXT$a}#2J4%{blU0wYQA@v-Xzt z7=3GRnf14%8h>1`)UuS3HzNOZ9eDEn<-N>*tLi@y0VkP%gslL|b9qY0Ck;2%tC}yOJubiD za3y$}`7u@VZM5&@v&>sn?K-sQ<#Wu>soE9b6-blXgSLnGD`L#YRqZ7D`wA}KK+gU* zT~JkTBkl_JUo*;U#Sq8;u&VtW^0&hJ`#Dwn8RD#w+wxm+i8pQzdt z%5!DT@RAx;|2g!ZmF)kLjjH~2G}cEWEZ?K*r-@_CCslnG^Z%o9<_oI+>#+Z5w~+fH zewz88s`^_|=-+1kj;jAQamnx_Evi0``dr27Kk~Gy{}YtYDo#J(gweO&MKSF#4TwwW}s(+K@a}FbK^si-6du!gQ>hD0_ z!ui*H9L)J|{&iLVJ(9COfqcruxBj=}nA!gp_Q%%m>K0D#O;vjo<+Iu|Y}@l{E}!P_ ztJ(v|-)fuwJF4~s>Gv4@Xp5@ttp+EUH>##}XpgJezvy<=iZafR^K-ArB$ja_9VFAF!H;4F3fA#Uz^`G-9~Q5yEW{u&F>oauW75QnT7vr*gtS> zgg9g3+xELA%jq3g^>b0k?=Yjj-$r??DLIVui8auev_6GCcsD>uujKk=EYo{E>e7TnMvuv+wh!E!-`DXLa z6H)uK{;X>;a$BD3Y*7Yf?1QQrXDn(Rm$z*{>k=#7wb|i z|AJ~*u?w7b80EE&{a^My)zE_Wv@UPtfq!?HxxCgDnXP}bZ0~PXgNMz>x{~4LOH{)K zj6aWY{+91j4ZD8{%=uaVhH7X>c|OMZS^kHr;Q(|W>*4ss82=yR{4M`0)zFFY>#<&z z|GjGH-UR0SE+@w1_!#F`>qo(ptWWWYIsbwD2FuC6Es$p%UJ0IdnC#y&yy7v{5F`J% zJXXA_8ho^e$4V@}q8fU@?~5DO`w_%{ALq~RPu`aha?W2MPYQb}q0jjX^f`Y)dYr#N zo)Ph<97cZLmvxx@<@^Wnb3)GL5y4nJmN6PqYfkf<1G%8+;f=ZF^5SWcbMee4wKw>nB+YUlRV)t$$Jgk z{lViYhtYo@pERt?3+?Ig8$v$qF!m3R=N!iR?eSXJdXFAc4U_O^eZj~zW4>KqWKOAuG{(pEC1%Vobe^+*j`=^Th8t)f zn6h>IT~(q%>mcn|oNqfc_qubn@hg{s#6Zus>z=1N(VqUXO3M!_4#92J0`+ zXB$dJ-cV8vdDP#=h+*u%8vX?3v621JJOJ)7@|Kf03MF~U;pf2YZ_971hQf!y>|e`| zRl|2s{u|jp%~*eJESUJK52?rva@)@+BjqI;IuiQ9m^lQ`RymBM^ujPmF zzer;LHKTqud8|+8^YC{Q`)|)LH?jY=d^g=N`nG&GWth3VHgSGdUssVSjOUxCjeK=h zMY8aJ(=CV5zBk$Y+)|O(p9aseoEW}snlrox|97J4i1{bHXgpu@jEZ~?y8T3;Uve1jDaQV-ZB-3) zzS0^C%Ucs{kK2Fi5Hpugt4*KFr*%r`=Y;+(W~(pCtCjt+{49l^<%T5HvA%9s4&!-hJNu{gw}7LL9Oc#SF`rS90`@!Y?C)dOROH9-r@dFmlgz)aB6MEY zJ|yI6=GRoDNcN|MJj?taRMiR0ckQTRFe$c>LHp#d6D8N1xKaD^|z<$|M6>CZY$>CZWg z^tVMFM*7O{ecL_7|PKtly#P zV zzZ>PhBO&A|!Bd8d&CQyV$=#e%r`# z{#rc+f4kUU&6W7)rD22mleb>g(D~*w@R2x>_b~rJRo@~x``gIrJ(FblW2%~-pFP9= zdfQZW2K)VI*gu@3)KsJWJY)U&0bV5}dj(Gaf~sk%0hgF3@#-P;QC!BqPpcXa`c&uS zpH07;{qRxV1bff2zw39X>Lm1^Wq;PcgjYpTKhJV`tv`!bACaGD z*`M`SRLzJ7W`8u3{Dw&n=PR`(oTog?>EnE)_Q%9??9cD2+BwwkPWVLSgY%Tyw^05& zVN)~f@3iUtp{lEHhCIvh|54RNh^Gxdf&ZIm6t;He4BPeE&b-6uZ#!>0Oz9UKCOP{X z#4if@tiu$)#FzdeXU^=ap~J;L8TLqcEpyNCUKvRT!| zP+xm8Mjy{x>)KHsdu};=23#=wv29_Bw&b=@Sl z>61Rmx&CZ^_j3BSy!Uc>+5GOkZR{;AsJbNj|6VS?4fwx7rr^uoStH-jsp^JsUcI-( z{41*N4ED1)!qer4^XfV}&v}mh-|%5oHwk;sd8|*2_Vyg-N6Ux7oFC0`@Q|aw8Jsa} z^Yh#^GuPj9Ic9D@&vAY?`~zMM#`x&7e>e}Tn_de}hVA*2VS9c$Y|pNB@q)XrF$`VXPngIfq-!Ke?~lC)?Wxn*qb#KIjAtd;2_xVQ*i| zVY25t3|sqp9433c4wJodj_HH{&_U<@L_C)zTKj+9{Z@+RF_V(i&lo#yn zhfct-x8HLZ{_T%BO!j<-$zG4cWUtp@vM0)K|D+=)dufNs-VKMz-ju^+FY7S*H|;Rl z%R5Z=ZaYl&?l?^LMEUKXbL6mhKsgM12k;Hb3-%5`Ct%n+;5iKc4#XTLd%nYDug77s z*XuCZ6Xkbc(vg$Bw8LcYhQnlU%3-pXb(s8{HoS3zs=I~F^nqIrN5KW=S5@7&Yr%Jz zhg4l*8Mx>$#^(dG4wHQ@-;Jcd9X#jAJ^U|+;4_uq#;mHN^YMd7Q#0uk$C$sN>b}O8kSHgPi;H+VNUW@b9L&{;qKh$E_p06H?IdamE zJ52gLf)gS>msgP9kkC&HzTq&XmvNZV%Nh>+n-=n%kl!}EDWmF&Sg#%`2zk-4t)D}) z4x{`Il?2Zj-t=!&-S<)chm|9Te}|(EBmQA7Piy~hOvvMoJ@|LHN634bc|CtP=`hyo zhld=-`sr|Lo@qUQIPEaWZ!q(E{&2?dCaf1{Dg7yCZZC(k4nGa~w8MBFdN|MgLsj=< zieGRT?X|}@{1n!|b11JK&d*clRV2CtZ2jl+_a5snG0LyU`fKIvuP#rLv%j`{d)VJV z&i)4Sobe}+-*OoB*TeY<FF(1(R)C+lsmxGIjW51#5GVtYvlEawaUYKM4Q&pFa zD)pjb{*S8ebDO{shtd9DgbvL&F_NQyyyzKjtyguQK>fYQ`Pb>;{NzQXN&2mxs{3Lk zIAPf4|HUC@-ru~KV&?LCaZ2dtg#Im|f7>w5QzLYq`Xc8a=cy4oPdyqja?Jx^>wlYy z(0S_79wToh#``Qslgz^^(oXUl4x_OjW&c{gr6P2GdX&>|C4G{YjK0n9OPs!~@0X%1 z=ls4D7y9f^pq~)>lR`hk%;l->`9Xuie|LSpiG_i<|56cJJ}R_EXjg+*<38#>^zlGn zb@cJyby`e1Os%ejwo(jT%v;@l^u4}}%DZ_`cEO?+DLecHK?^3fj;e|w$# zzP^itS%e!M`|W(6?Ei|RPkQw6@clj9M_q?L=nMKoepA_17*qv>+E>FBDZywzx=dNW zg7nEYbEOxSS0;o!E94@6l^2#*iTG6_ewBz{E#g;;_|+nQwTNFM;@61ywL>aYKDAUH zRHvN(TCzuF%3K%2wNsvTS;2E*j%2_X2Wa=nx8NMuiQm)`j^C6F>o;Y=7-wyIO+{Qg_Lt0po%YcTr04!tKY9b#j{eeHVW!_J zxpGVqAzqOU^U9tuuS$fuWh%^TyfCj7`g#~}{9BiXzLVc|`EdOA#lcQ~>HSzYzfTs! z`Wq$%=fk{-^6%6iz2D}R2faTQ;$oO%Enw>F+#X{heyhkYy+7sJYb}PE-XC(?H@!dP zw*RLiV5fYZruuNwr}uqA^69X=Jqvc)$Ce`4DSvt&!A+muKXB8h_X%A4TeCv{c33}7 z_8foX-C_NBQs`&Fu6_YbHn@E0dA#cnJzsbAw+)5$>G^p`pX@pIJAkfzdfx5o)AQ?) zKGmP&4?Uj_>63qs{p~$qr+n#ou*;p(VctRZ9esL!>GJMGnCX1qmDBmX%X_n7rt@T1 z?k5G4KQyOvdHZ*8?ew>Ov9Nq!uaJxQ`$YWbMSMChbnQPc;?sGZE2r}`*FQQh3&}-% zIn2+j)53C;^H2rddPg;_Uqr~a#=LhcEU3-$#k1SbWj1g8aO1ZM^31m^`81Q!LD z!mPWolmF_dkb8pTf_=dW!AZd>!D+!6!CApM!Fj<2!9~HPFzfBElmD8ikb8pTf_=dW z!AZd>!D+!6!CApM!Fj<2!9~HPFw<1vj<>Z@!Jgo_U|(=Ta8htea9VIia8__ma9(gh za8Yn6%=&o2ssFmDkb8pTf_=dW!AZd>!D+!6!CApM!Fj<2!9~HPFw^w!)_)`_*b^KV z>LGE($J%nU+>={_CTHJ;8CozTkx5q~Mg`wBU^3tl*sB zyx@Z1qTo`PX({REzac8v6C4-p3r+}53Qh@53(g463eE}63oZyQ3ND42mfCLq8>50f z!EwR9;Dq3$;FRFB;Edp`;GE#R;DX?y;8K|B=)lc?Q&g}gI4;;1oDiH8oD!TCoDrNA zoD-ZETo7CoTnaO7z1;jSi3;`v#|8U>6M~b1Q-afiGlH{%bAt1N3xbP+OJSz1otyth zqJllaalyXegy5v$l;E`BjNq)`oZ!6Rg5aXyQkeDmsx$vLM}^!I92e{hP6$p4P6P?bn8D#H=Oc_dV=GEeZdLANx><>X~7x6S;0BMdBFw2MZu*o z<4I2_ze}TnJ;8CozTkx5q~Mg`wBU^3tl*sByx@Z1qTo`P@gy*m|7B6Zp5VA(UvNTj zQgBLeT5v{iR&Y*mUT{HhQE(~D`uT{{-j_#(+!GuZ>LG zE($J%nU4G1{=OnA*b^KV>LGE($J%8Lw=G^1m`F*b^KV z>LGE($J%8Lvc!^8aX5uqQY!*cY4-oD`fAoEDrBoE4lC zoEKaWTohaiGhQJI<$qOFuqQY!*cY4-oD`fAoEDrBoE4lCoEKaWTohaia|_Vz|MZ)f z5PO2-f_=dW!AZd>!D+!6!CApM!Fj<2!9~HPFs}x>`CT0q><>X~7x6S;0BMdBFw2MZu*o z!D+!6!CApM!Fj<2!9~HPFh2=&^Z#U2uqQY!*cY4-oD`fAoEDrBoE4lC zoEKaWTohai^9G=s{|!;Wp5VA(UvNTjQgBLeT5v{iR&Y*mUT{HhQE(~D8-Z^AH%0|} zg5!dH!3n`h!70IM!5P6>!8yTs!3DuZ!KE;70=oI%6cy|Vjtlk$Cj=)2rv#@3X9Q;j z=LF{k7X%jtm%{uM(9Qo-QNf;vTU4+oI4;;1oDiH8oD!TCoDrNAoD-ZETo7CoTnh8kKsWzS zM+JL=Il+0s1;ItZr7+X$y>9;7qk=ubalyXegy5v$l;E`B zjNq)`oZ!6Rg5aXyQkXXb-TZHk3ibrY1^a>%f|G(%g42RCg0q5ig7bn4f{TJnVcr6C z^H2YNXNWz)alyXegy5v$l;E`BjNq)`oZ!6Rg5aXyQkb^_-TZEi3ibrY1^a>%f|G(% zg42RCg0q5ig7bn4f{TJnVU7dc{Kun$J;4;gIS+{Yf)j$1f>VOif-{1%f^&lNf(wF+ zf=glE26Xd7{~lx@OFSsDMD7X~nZlIh0 z?xVOif-{1%f^&lNf(wF+f=glE4RrIrJ1W={92e{hP6$p4 zP6ZM#r^g-`TPLbK@#C zIy%yKp@00mxp|>~?7Z4CGIHfYU;mjiW8=f;)RnHj(X)t1e@BJ~1}}~enpmg%aYZRj z45%&R6Jrzorx9Mepu@4zGyN0&fg|eZk^Kjcy>d`dQe7%FHh8v6UF@Ie(wkM}=e>SD z3IqJ_TL@DX&HtY7+>#Q9oqs{Kdj?x3ama|(@R&DW&9*5;4@`{;IH+cZu}0uHh!RBE{@Hau;UMe z!1e#PAX}vT{<4ST3{s|km0khWWwODn?C*Ma*_1E%jxXPLx zxPCsn?@3=vCtEs~q;D*(_Wju1w{PFReLv3rd3}ayS+{`Nm%mPbX4jr0Kc>4B&!x1! zzjU=^97lc(zxLsC0X|=sTx0z6&Xr$@u7qKWX?#dxtn1ymFaNVK)ru6GUhrKuH#b*J zC{8{f!O!U)W5|JYJ$wetui+c{VAczm^QPvASIhiH9@!o0izu(*z#UL z{w%7PHTmM^HKiMoZ@7hg=c9dHz9)d4jeP$peB4awG+-6v>XfB<+Hws(_#}q;!U~d=%Nt2v%r$HuZN>-YkbQ2+C`um37*_iNBU-Q-9NfKMaHI%U86?U+ZT_p0xbq+P-pS3;tT3O&x8)U*|(m z{$!i{wSIQwG0UIU%l=>0g1?q$Q;)RZuk$Y`U)?5ut)Csa-|}B%`Crq5zm{iHUv9yF zv9>Q~+T^eGvm1ue{oB%hCO{>rcdnF=_3r7p6nLSe?T*HJ#a~DLGyQ7Smu80k`IGob|4hrO z`@b7yqSgKL4xu0Q&pS@*pKp@-adV@VpXgd-=mFRdWG!HwQNIv3>K}t+HwgW~z8f zg+#s^z-bJ8nHJ9XW5OqHRtlVWVuQe0kfmJ=E?Q)$s)f`4RZIT@rT_S6l|I}R>CfDO zFC_M7?pmz#eF^DjqH93%$(JeTsa^QDDc-()=(^zSw+Hm*s&jLLu|d;+Vs4J~={Y-( z3v?dE&3iIGa%r6xr}Sqz=%#gkjP>0w;}f+r&Y!a#=c`k83Ix6ac~HxIU%`ntiX>ikf&{psh?PRsumfinkXSWFM* z>s#RK^q2QM8K0nhy}&TN+~U^>oLQ;uc0o;?>S5-il7C#)W^$QTg6A?x3oo^BwZKUX z7TawE+{yN7otMSU>Yu9{kcl*la;`V|w8lN*0gZF~E*&VpN8>%8i#U=mRGas=f!%k6 zo|R{@?k95NYgDey8eitpR|_5t$uxkSrYsIYkV0>T#kHfMr|WMpqz*6iwBNyy3KV)5 z+xqHQLvPUHT)jw7*YRLT`xAPqAHS^YOfaP59eS78b{b!Yo{eXQG=2`fkK1z9N9d`( zgCUJ?L+>(MuJLK;eZt}ze}>-Wf;XNEIK6F$I#{!MM8r0|=w1@`QXN%%$@vPyRSKQ{ zYV8{8&7`#6BqR%1oC0ZdzxP;X&h0t`^DXmm-bWiUYB+=cHX=5 zs$081EaRBy*7!Qn9h32hn+<4b_cMUEcFjos(XJm#T%PEpqJ^yI5f6`kI3IpcAS;~2tmGcsnlY@*DwJVNeXze;~ zo|JOV>pl2I{bZzE{c^Zj?cWbL8_?f7Xl=&ahS5v{-em2jnBFgT>)Z9jzk}8@Up-wv zd@@7w_i8==9!vjTr9Z9yC6gOJEOZM;biEoyKG)G%z$Dt|#t%vPZ6HJZrYrD;RNVYc z&2N(PLCJ3%r{ghgM?b|xeXJ%_ImFLh2)tTiDf}Di%eq~$IF$N2#+&Wg6SscPG`6Sn zM{;(P_kJMqOrw$9Z}?hHI`gl0^|+m+I-iekyhjdk^X~%l2hP^-YoHL-XD5Dtn?!@B zt{c>2yYt4KBSh<8jsCefp*qRl0*{OX{pNsqL`Hn!{@{eww1vkkRKHgA)Q=TC^;<=I9#|;p zIE{j?+gGfD9MO6Be8W8A2_nb!LO|+U??>zYghSD}9zJLF{ksBP996XOWKeou`a?N6 zKgMuA1Cud9>F*^!XYU;bV{tv_Mm_v9?mXl66c8k~m-FseJ&va0JD2wcEAJ+O*5CIW z?LPTF`)~2y#&{{wJ^McW`dGJ#QDe5=IdSt(a{dwBH@pJ9w>tl8K6lw6dWg0^{k~(! z%72-_iPFo`-uVR=$BjZcJfC&*teDYzpj=w-a}pATq8#j7Kl+_Ov(^FC}D1T32nEtB8Ul2G$V?^2orBbiq=KCan68wnQ&ujKeKK04@;UDf3 zJOPImwSykW!*eCx*HEr}=6*e{KhmbDhJA_oBOp=znfde2I!_ z!fO-{wyHjB?HcOM*m2?V%^rugeXv#a+)g{%Nv+*BHGu7qpK>l4#TU{fvbp~W?K&Ox zpHD-6|FE4eJ)~CON3k6>H^W62iFQs7TP&S0t^S;LeJS#tolcZr5;HlmR2r5rGUxQ2LK zR*CvPdK&e(pW?om<;3>@5AD9ZP*OiddK{9%AzdGNqlmP@09%Jea#m&ANIp~_5IiHOMcW|xQ@6ugLcTo z^?x;?oyYs|`$C(~NYpP`dzaGztv`(X=)UI!=tlW)Tu7fLbmskzo(H;kknQH%=WLVc z5qy0RCEc$nWPHuke4cl);2 zZg*d5qDTEi<#FdI!>HLP&w$bPadWrO2}-K(qCNK;izOt^u;U9oo{;gF?;e+S-fCRV ze80ExEbnuF^fPuKZr&C+QPO!ci*Xx9e%1Ot?i1Gmr@i4P))Sq-v_mFqhqe6Om_KyU zvj0qCy|Q%t(!HXGNf4o{bnk}T`ySf+*N%@Aq5z?-fWo9GaZhDIJZc;}VsqeJPzz^{ES6 zt9Qn}&sCdtv!f<2TXen-oSx4#XmXv3o8J&Vekm#QcNIGImxZgo<(Ktd`pFb3Au&Jy z;+y;Y@@lCU&A;-gXP)!Rmp0{J{^lzmn_-ZNG>Kr2@%WP#Z?rdmsZ=w|Z2Yen;uISX z+_G15`t3P1@A;F+%Gv#C)ShXVjr0o@ou`Z^&=pzN=zq4EJMj%adrW}gJ<`1G%i6wr zkJf9XGa#p0pi{PVKzu$r^b;JJCiyIV=SS-6&i93&i!-ZFSo+Wt@4E_iAH5y@qW=rZ z%VZq5@8I|b6@6#HeGGA~2ONhjYn2Y;tWEeCW`?DmI~UAn^!=V+(Re$b*(l{XZ;L>- zW6EbX3m#OK3k);YD@^OV5AF{crx!QBs^=#gq*?8qM&X#Q8{vT)q+LF}L13$W`gSXS z)&3#D61(pY^jgul{d=w6gUWBo8ij$Co9|FC)Mm@gHh&WohO|Ad`l-5hTWvY=nSzZ< zfcmB$i;td^el58{ip?(6JD2qiS{(Ig@>>}ABh6TOw@~Rwx$&*i-$dWfWIdfYqT|qi zv6lPpT*l&svaZ9fe8RrBari0cLZIyOCt>uItA9Z0_FFk7A)n-1d1ih~%U%7^xZ0I# z*I3?f>mLSvnB!dDQ+tTUan!RrpW27t%kdd74+*2XU!M{^v@?G=e+SGP(jNC40vqR( z^CvTVrP%afc-hW>SaJ6bxZlP_jB}DG%4OPn*6jhaNXCi&gX7ZqK4Q~PDPi1PCvdEHiJad>{TBB@Zk^+L&qV*lr4#&# z%05Qp>h}Bw+pGUl$BFo>Mu&bfZ{NR1_vwt!qjH^opH3>L?$d`G+Pn8TT$dZ;^t105 zckR@x1{iL?KZ;JX-(#_N#ZsdEa=?Y~G(X|NV@-FYu+h^n)67{inTn z_zcLaD&#|n6uEPI*SMwCzGE6nWXG#J@r4vOFTO)~#`jc*-!&2t| zVG3VJq4$RLKQ7Ur-q-IyKKTkH9rV%N>!UkzzjYL(7_YM)q0@^yhm4yf$?M8#o5B$tMWdNe)MQ}x5SyO&v{5a@cqYN>@Lw< z^gSBeN96@-T)gGmchCM>k!zm!Njs(gzO^@hEoaMo?~`=jUO24ryMLWjGyVpx?~kxo zh!;01AM1zkPD#)43&;9|&nT*K{HmO&NcHzhcS?E3=lUh3!*@sY`^QzSb^k`=L!y5W z2!F>!_B`J&E*3Kuy8;R5QTJ7eMe3I)%DTUrC=H8z{V7yJnkei3u8!;eua4`!u8wP* zU&nP{SjT^}fxgCh_4407rQF%!q_vMx2v5Db`is1TCXW{b}o)CDnuUGt4bbZ|q{Y2$n54qTHu9K8o*OyP>7xE!DO{??sG5vdGyiXN5 z>U5?AZ+~C+38^19X@MWayS3p4eVi^HYUkbB3hE6?qg^$|Q}dmBu3|kqk6XW6dk7<; z*o?t9?zQoDRaZR7l zq5V-0QTuzgLEnFXa;LAl{ryg@zYDX!*W?@~ZuIXM3#F&6+_M6Cf06`WH;>qVGIZ~f zh%@Cpy;t$=J@bscR~}q+&MEgDUwBCCbKf)nJ%_%pAYD{3F23q~UL6+6TE%VkcHX$|C-@~18MYk4;HYzzMC2ZHjq+T^eGvm;Mg{yb&I_*MR_1%EBirjALy=sJ4{ zez$Qv=x;eu28`8jX}{3jg-lw0P`*!a#$m)q0p-3R1q6w4&nCbyt#L=_y&&~@4#e{W zw{A?>_n`fI*h>99`LaD>Vo^-*=Ycat*NX=sw|_+ausdIN_1)i}7LL3we2e2J1O`X+ zU1B({aZ$Kg-<{?UyeQ@Uf%0+hm2TOfc+)#Kdrt4p_p5q+YL@)E)(>`DNC${it9P~L zFS%Y}aIZqX7lUfdmPy4Y9#{FCJOMYDNZT=uspqPrbEjmR;42n3zH77h7W_LtS*O=& zJ>F|9!_O_D;{Detbnn%2n_m%qOrjp!^}BtNy3teKu}l1I+q?Jeljx7Mv-!D<#-nb( z=jh7%08Wm6W4SwbKMcMY?zw#CA4IOPu3oW^prq?I&v&LE=NSG-Y7Zdohx&B+#F@gh z)+as;+^yfNcPVh@Q?=3eO;g|-wYw9*8JDa=y;l1H()l+?Iv;)RybC40=W_cK-cwRv zP5Z5i-EUd_PgLxFtA<}M`f&R5E9VNH&pan^qN4i)S3Xg(`>$Gi-AA}`cds^4-l}v{ z5Q=h@KcIb>uk@)ej+cvf>3^2vQ+ppR`sDxDqW6r^@!~t++P=lwXY)DidSS#k3sYeM z`<6rY@2DPYc)#_Y=qc(?Pyep$8{GRSx7wDR!2Oo0zwdYUdG7Pt=X|G1KeqMhCl)SP;NrEw{b$#r=;~bd zmX>SUJEz?;_{`nZ;i+#?%P*gD8v^fMzr;VhvM})Iir_zY<3Q7$F@GAL+wl1X?~3p@ z_4udWg||9}cm*Fuv2)=+zVVqqckaSjFLv!nx9^QN-nejUV5YMSkFP9tjwvwSetde} zJMkCpNUR6?A@Zu(5h%SY}G9QFD9hBRcAQk__liIC^11m42~3|^AHqvM=1)KP-@WDcz5nSO zkx`~|c0&k1C;cGbz2bN0NgoY^;m>Tm#`J^#-CMpY>9>XOXYZdU{UG1H;@9R$KlF#` zGn=k4{osH1mM=>BY`rZ1d*?|%$ak;!#d*>X{crlr&^4wX{O{iKSxLWB(tlu{^n-l& ziqFiGepd*8X7e?sAN=p$@=-~@Thf2uJn0Ad?iEMoNq?!N|KK&IAN=p$GAZfbAn89e zPx?W=d&Rza(&y;J_V@5LrXT$8-tu8df4QXp$UNx>`R)}T^m2}OyiW9A`DfnAY0tj9 zx{1?wcz>_KokQ7){*I8&m)~W=H+=XLAL;S#JKG;wyhh?<3psu!gj>5Jv_CyQb6mZU zgC|@ZpEvQ2ZpCn|@vq%?wAT2%%crBq=Us0)K5zH_+uR5I-HtoCo0z(DUq~tIfSPnS7jd>=F3vDip zkJIO#^zpCkSnO_OyoeN0yQxp&4=$$r7*g7aba}n~bl>gMX>)h``@~ls^Y_;fE!)M) z&Q3>Fz5V!am-ufB@qhV=5I>p=gJ`+n=q^S5R4Ocx;F7z^B)(@&Q}zu;}B~?U3!+wl9Wf zz4W?&_Wh*I{X^(4gQ367D&L*xXM>@??Cdz;>vgpMQ$C&C>8JfY+9%8hul@ZmdH0Fa zkNf-n{*(T`b0Kp*+a9W_^0K`QK+gV#N7dhk7|hovd+ccc_4c>_P0IP_{?2@&O5R3$ z{~HEFpZP0)zji%uWiZt)`n*1_r(JlvG3TS1WB1H$7p?yF_Vo38qOVtszOGmA-w=IO z{e9weHMD~Tcl&~{9@u0G{SEJa{B&r4y$ixTZuP6j>Bd#sHQNm@wr{qdRzLig)^|6H z@qM(S551FNvG z`5oTpkC=7=AYEAn8c6`oobctfzm3Y+isJV!4ZeKFkgC?0)YCS5Mhc1NbaL3Pu_IlwHLH<07!C9O06Z5(Drr^lsPFTFpL^!~)(S^w0>{{E`huWm=y9%4^s`O%~t=XqY#19P;~x5nA_ zl(rMt4)y+$4d3e?Vszify!E?MuCMv~g>ybS*N#q-LVK7v{rA3nC;G#@&T{kG-yhtM zpAPf>g`0vq`!D;~iT=w$FS6c=`z>ROmihg#-bHC2pm)&%-+tTNDyk>gN$ZtcZjVELpwX^>pLgoV(t*N zj&Tv%N^svU?efH67*}(jpcL_ZGWg5=c!)3esgRE7XY=R_FZ+0pe<_U5&!gXR{&)@d z1mQ;Jx;KmqMq2#d9j@i@Q!qNeHx!T@VRiBgt}Sa z9PMTu&t=Ag@GJ?#^B(8(W_VUAuGnt&_y3L0&nxik_qTIF&Umgq&VRq-=W1ymEB$@{ zOW*If874ffhwB+PcY3R0cE6z>ueIOo^t}Y(mEO3*jq}}w8PTVzM?c=U&!Gn44}afY zKXanT*I#EmD;NIQ0@_DRml03hO;_Yg+T-(R4|OcrcP?J*-8l}sUjr|?*E7dac)>Vz z#OLcM{-(fe2W*o22SY!64eqojR^r{ww-JE(=bi15Ge6}C?s5EWbMFSRy><9@wsXO| z7~QOP{IqJf$Ipg-#`c@QpKbq4kLlB$`Rnai5ca3QbSKj?r$648i#-(UC+>$d>xYHL z#r_fMrOW4!;on}p{E4rRHup#V&Uko5eURzY=4Z8#&pC2<*e~NF{p;~C#&@Fsyw~T6 z{)*hs$vuomC;G$qbD}?tH*tJCemdy)dBhv-;CqmPtJ%R9O+OffKh_J!A?hpG!L9Lt zs=WSr?aQLCv;H3Kh!bK@O`pE#%fxZe!p@-n82w=R`m7f_va+PKke@_UN5FzSpKyerTzL=Pz9UxohX+wYsZPq54DL+=xC|8c(Xz3!NlW5(YnPXDP7ANzN- zW1dGoxGx&}%ZbflKmWvF*q871R#+$p%XMNf$d8}79O_5r+vzhk&xOIC_4nyBf8_6T z_X~c~yB|fm)MsnI;G}n-IP=H;9`_%PpAG3Pm-RmN#(j<`-#_?S@1J^RKRSLkv=`cY zx}6jTf5!XEEfPBe<6{E zO!$S(U-q0kdnveY4dp7la85Y%_9b5UcfvtWg%`dmEc+5K{8z%V zFY&@}1b&>o7x{eZH$C@-YNcibY(DSu~iS?Lg*p`40TSbrjCD{|hqTYI~%D{2zZ=`paU* z27iNL+-AF6<2kgyi8J5y`k~!QpQD3c_Z<5p`vo-}#})3gF-P(J1Lt+W?#G?s_iz4M z2!9rUs~zs=k)Pa|dT?JE+>fF>%zy6X*Sve-xzKJdD7n|ebD@1{cdysy5CN_WTm8+& z_zKSk#qfm2B@+3T+deVp;m ze!hMk`J8{ged!$i)31EHAzU-x&GFYJqj^8mU&Kgjycd2hbmuw2wP zi=^8Phw7c~Gv1>Pe|_t_kNfbgc6VS-d~9DjKXIQV4WBkYy)J}ra|IvI^y#8+2XS9) z=Yre3ozZ$K zxG$U|;EL^*2f7J^Y*EZs)-q4-w(&3)K?$e0> zqq2U~?Vjz(#((^5X!o2)P((j(BMQtPAHiSb_ng1^_Ze3We$kJ}d9jkqDXN zEA_seMhY(4FI#ahY)<=daX)SU=5X#Qu9x?3e%#AP`=9oAhU2w=^HZLu&)(_ph0Vj^ zJ{aEbA3qz$HLfGP9egf~yM>pY^!`q44)Qo|@8A5Gcb~H^ek`QNanrB&_ zr`#W7Z3a8t`zr|tvyyR9KUC@VXFa;O14p0A`11B*xw0AZiI+n?#P`)v|1A#l zTI7kdLEn+%{L6Y~?~Cq7p%&(AIV*wpd%``myH-CP=?(5jkzem$@_FI0;65V#jpb*k zqfndoC-Gf)RKmyfgue_%^L%4}e`qh-equeu_Vj5y--%DJ`#B#G$H6cz7rma|s$Vla zf*}02e0ZF5_jlXB;`nmmlV0rYafZwNeBgM#a=F+cY@Z24qnocSAM=eby>7;rW6ro1 zitBWK*dpzDjQD+pf4Tf+eg1M+$q!rpzkYtU&6^)I!T$pJd2rtRu#5le>t}G@{BY;x zUq3%#{)z2}=KiGuHEZ};|J^I`t9~Fe2@8SbHC{CJKKBwzFQo( zSRSrB=!aJvzi0>W;yA=9wpX3cdwb(pn4sF+Hb(31LXMaF<2l%~-r4WdaepP|kM{4I zuag-I<|7q8ZNhU$mgV7;F?U@X470aCKEIdAL%&=lw)T0RM}2V|-nn2ohnrdX<9om3 zr^RkV`4Y&PKRPp3T3<&-viqU)+D7f0n-=pI(3ZC*+S= zzy9_g3O?_;4a%efE@bfsSDSmM?Tm(3Y(U6|FISL zgR9X0*S`F1E*MgLFTM-zY)Agicu-;S{W;gmd_VOf?g{)Il{?!9+sWFS;DC$a$x&Z% zpAhGxyqx)B!o3T_dzzgK|Bcrh!!3fW`TjU`Yd87@?n+P(clz-3$1CdL75Jl5o4eh+ zvmYXw=6R;Iw**6z`J{XAB46IN-bL(XZZX}NR4+~;uwQati2aD}LH}VrOy*OxYq^f# zxXbq#3EX3QVfyU1Yd4|;b8X z_7(Jn_Sr9XleF8MoCjvRNFZ;WCkp%V$6pTXhTNkJ4ZHI(6oB_FEIsazhxCr0!@>a9 z^KfSUUSJ2u)jCfU_T^=K#`&X9o(^@)h10?qOZo;;`;)v8->J{=@pAB@gS-76*H= z#bJHb;%8^2AM6rpHe9itMf>`B6oyi^Gq#)MQos8Bns*P~x%s`V{e!a3IUVAqUaxxo z3iZkwkL{lNCCBq)tZwRe?t6>B@p*2}Q_^o@Ju%#!`1HEf_>1_F{@a_ULcO)QhkbgZ zAF2EMiH}^8drj^azGUEY723+D^J|MO>kzFb-#X`_!fv%7ZsR*3d+)<@&*K70P` zF6vF%k+s|6xnZ8)$JK|C+xaMeGUE9qkYOi~WZQ#`hXzUMKp)dz9l}33^=Fxn$OR zjj5jt{y46s{mkoGz0Y9l3iSss?iI1)uD^f%y3f&k_F>B|{9pbv)3f_8t?&7leEFN> zEyh_o*!5!T`~9nYx~=c`|6B+!1yAn?6eqe`h{-jp+yfySLmb>4yn% z`s~lolYWrzUePm8`fNo^|MSXauX!^{lYfL}*-@WCpB>iBhPM`hddD0K^ z-7D(zq#x|y=`&xr#`J^#-CO>Xq#w>VO`rXhdD0K^-79MIq#x|N=`&xv#`J^#-CO=p z(hujSrqBMHdD0K^-7Bi|q#wqC=`+82jp+yfySMzVq|e=Nw4by8cAoTueD{iq_owIE zGR9lm9yrdjVfVUMy?>5(Tk%xr4|c`$>61Pk+TZL6(`SCt^HFq5^7FVe#qBHb2Uns0 zkG#Ac&5$dRIDbsY`OxLl-kTrinJ(=w-g`S2gz+?V_gV973Hd($r7-?3|Dg~*`oHk` zz{kII*{6Hrg{r?#e<_?_EW8lxq{36-9P0j${IQoW|Ad5lxx@3wV}a+KciR!%pGRR? z&I@7t#4Ue+aGyRZ_Z=(+^b!5h2*b5}INYZ{74BRgut#m52|V%TkiV#xX(|*KzZd4y zi}4%_{-=LJ?)H4!bHSb8FYCeMblBeU-|wX5v%L`H%O4H*?Hj{=^r=^U`R%&S+TFP@ ztj;>8PWwE@7tgOBMfsS1bRXRu?A5Rzs_gw+Li!_;4?lk(J{vx+3!3=}+cxgYoUd^0 z-~1na{&y~H_w8u5nuFOk!$#m^Q0@zddboPMuLJ$av`)@{Pa&-Qr#`#1lAzfYW%b)3le56b%SOeVy;0-0jI&ipH{ z=W-$4SdY$T z^L)VF2+w^FmtfrB;DKxUD-|Dq?sPck%6Qp6qg}ke|C?UUai9A{%m?LhJ<;ZV+1uZv z@_kvY2ll7==cQpLkNvL4{hjj@Eu!nUn|FymcQosTMcm*2{hoUZ3ew|28s*XJJI>&2 zXB_h8t_!Ip)Yn;m^Xp2ECp!FH5XJFjZg`dgayC|6IvmHN*iSBe%8O^EANJFOy+*yh zz4pFV^cub|Hu{n4ZRf$h(&ulO(;xJ{c|3o^@dHKm{heW4QpS(h}?Z?pTA zE#0ZMHhgeJe}S;=EGi}YQ5Guqwqj_h-kbkbh&_`HA*R}@uzviRUyMvFa$!{frxBPp7l zf9*%M9q0H_F)d_&--i#v`EL_U;aQG(FdQRqV_Jvtkzv+-9M7~Hc%8)OUH-|r&xLyj z+w3oeXaABz>D24`R>)XCjQ2~yUkCT~zADdrYw)%3>}LR5R@Mo_Gv0q6{B_tg-YI_Q zPp*Y@;{~^7UC@5z#k#l=pEN$K3z~36d{`H(_bNUY@xiq$!Jnw-Yl(~Di})}u##_bb zPw~OE>@NI?ap7^{O1g$?R=Y$x{(|FT*27kO?g>tQRmA5sK7Wo6pNsfh#s}B3RrsUR zSnZwtpIK>yQP#UJe9t_7s64qAuEoRO;sd9}Z1?{!K2gVf;m79$A5OZgvot>VN`&9T z&EQ!s{5od17*B#nmTe$7`Fdg3PvgUSVLvP4LtU=Jr;1P5^>EAX#j`Fa`yqeo5jtK> zf4Ac~wv#*MIf46tJg4!zR-UtXexE!S@%*?vm+|~V@?6F93-VmY^O!s{69?rv!Tsa% zoW}h#@|?x}XYtv9a7Ent4PNFgS%|(P&uKiLk>@O)zbwy1JXhqojOX+6T*WiLGs1Z4 zc>ZH~W+#3X&s@eNc>W7{F5~&H#ZLm|e^;KmCgp$2vOk^5uCL56m;i?&T8D&TJhSf!pB?>)qjlg^3prj_pl?x5Jw@wwxxX4!)T1kDEE3 zA9DwC$415u&K#N;E4ck*$K8?K8;ju6|b=2)U zlzVZbEBKu34f)yY^0~c^(h*7vh_A4JWY6J!``qN%L6_e@Ga{uswBJo19y>fXav+z_ zPoUKKLsK(Tqf?Xa$Yk!|NdC}R0mTn6GddAn;Rik^rVdTa97pkAgtx-T#J-7xd&iEs zc%K;^2@a!s+<2}q?%*&wnJW|~_PV{fnVkOv?e294bNPu8xbDjxn3z274vig{ni(6J z$ZvF$Q=_>_|I~Ec)O6ewedejN;MG4e_K^c)NMZ(6bs+DKP8=E=VbNbKOpO%Apb`H- zbp{_nALB>G|30b?8ZWrQe*D{m9L}JX%#6A{`F;0#SU2iAC-x-+kgk3BK<&RI()XJ~$j2n>l>wV1h_S4vp=1w0d>$Oc|0MnVib)9ochyW~{(O)+F>f z{PfmsYm!wH`AusQHm9k519)D|plFGkHYS4ylMp*v1Y(%le>gufHaT`+V3lw62+KQi z=1YI@dXiKr`UE?X)_WdG?xSX?%7NV4hYO(>eQ08LvMM={E9~Ed$JNP-hZ3~KeI9&H z@7t81J|(LMXQsvrEXTTh?zk^CWXvMF9Ri}c!YAbi$Bxa6?>!{B7s*4(lbOu3+jndq zdE(he-~Y_kZ978sF;7h7!KkTykH*u)IYy^)jIXeaW7%R=k z)hs3ivG`IP`eA1BV~6&Qj2*!UH!^j2CV$v>Mz(Nt|JW6`?0oiFw1|-{nVrbbfbS%$ zeXDpd*{~j-@!YIkszM6AVZAg^KL|iDt4dU!pn4fCW)2-54T{*9U^q!mtn;Hfo}e%P zxF!khBslpd1E-0-fhYEU42c8{P)i|N&u=v(=q9@wfYD*q==j*^{$|lUg>L(?WNfl9 zmP9Si9VnU;4`SJmnmd79`E0BRu>^DKW*%h%gyP(w5iqIjEk#*x!7zrm1Ot9I( z+sw@5npP_LBbzXwaBzW?1zqPSkI6lgWVZB~sREjG>e#;3$%79j8$X7?oy}?tne$F+ z?2^oe0iKO2c#s)avVmwHemL1k(d>bsLMC0 ztyX8^XfFzkJrK6v!57D1Sz%HN({e1_tf_oWl8W<09L-`3xxGJnxG*zvXlm*}h(P9X zBqsgf@uxB{{YFRnmeC9HqX;4Kc6N$? zaMOJ8abr$RafXes9xa>AE1x_FuN>D}F~y3X9LW2w*ot!_@;htXo};75jZpP$yFzrM z67V)>E7lE+RZLLk8qIj^!(tJkAhD&hXMb=L{dTGe2wuY!Z1tr5Zt9vGrpq&1hW>`O(V}dxozZ`9Xq|% zakcJwB!tmjhgN3G#(I+$s9JBFch@9CvsyLu8>ePkl(A8BHMVaO%XgF_EDJNwZQc3! z_L1SopQR}}XG*3K|A=^f=Fw+AF!Ic!+ur~5*2hOu$>tENlweLsE(;z>&{8)izT-#J z?y6NXxe?lrlF%?iUf4)tjrfeKX?^-Imh_tvjshoWr80%tcEej(4Xp++wQgecKpu&$ z!c)H4s0cJ&V_#o4xzXDd!8_7L@0Wz>wYiGoicWN?7DR7rp@arBYsKl)@yf=Iilpr) zf(^}C0Skd4wmYerP47$Yy*J6oCd}q!8`i7I@K|o|*rBF{)G}-fheoA^&#}RyS?jal z2X3}uYJCMlg^bQ!mwDgOnDPOzGMn%QNL+EvyEH@e2dUTj3?t5Vp(FZK;>s~u8*41u z?Ri)ftjK|o@FP)va2?m*Y{|?S(n2dh@iCahq_v?Q5}BdeL$+GRv!ox)=O~X>`-f5- z`v;nSl;sYuzIt0z=A4eixf4BU_Ed^Idyg@xR(N@eg1FopL>0j;n(jXTVUkUJzQWQZ zW@iu`5;^u+!yHL9`Brpj;>GcqB)4y7>`(%`51Bf$DH-=|X-#tY;3W3!lAK?6AHWJ& zBrseKj1|HvJG2*H3&9Fv)Nu>NYd&tK1QoZ+70a)UW*>N7Yu=ZO_B=|3*pNMKzs=IMS{kLZi5E{R2!8x_|M|6b)4wx|$uqFD+MJ zdk>L-)ZC`fQxHE#S?sbg38qGyZ@w-{s|J5GG&f{Kim= zlNPt;uES2#J9=}LR@Bj;J@|(O0l`X;s$h_r~hp_}$@Au#QYJeMf*SHXZ#+yxb zu06K-t=CemiLq&rH6uKznixBbH}+cVwB|VcmJ7BwPlT2Y8kdZjSmMW_giX#XC~VZ) zl?Vf2aGN_@sX@Z)JZ=qQ3lSS58{&>&=T%3N=n0d*-*UXq@6Oz}VYMGSAAcelogB*@ z+U!Tp{KVKO2E_xpW-!T_#-fj2Q44mr~I(o@+Pol@nS9Yy`>XkudfTVuut%)Zh_uM#$N_ zN~?mEfka}PjyB@kHRw_4SoU!d+1lUQu%5%|APV}x*oODM_klfJ$kNNec=98&{oRMv zJUtHGKelHyw`tG39@mS<(e-=Q=f*bNKd(nD9(;`J-@A9?*q-(K;8E;0CXO7S+1+=) zZ;PVn9C8+CFD&!pUOufGcz;ha!5#nBhQALQ$1&E;tewwj)8%~qgkfph_N}{~d=A^= z(uYuGMaHWT#uy2dsWX3!(dkl*?YFJ9u?Lcaxy z)eF}=+T@a*%;wY=_Ee$06O1PsM*i-1{ICfNA!BJ8eZf7vkWgwb_;p5eU7%A4yNBH6 zg#T6>`_Sx-P}K5Uj85E!!W4TnY?p>jsR7=+@rVxT%~?9If%pkR+Rew|$oTW?{LZ*9 z5+*NfyhFfE+t>?PKi4|3h4DM&+S~3hQeyZyGu+w%7$B=>TOPrHn0Ma0rd-`QMbA6;axU*x;zez#AVUv^FjmZfvCl4RQ zi{_~p4^9-Y&9<$zqTJkEXeK-1B?qHv&*T`QSH_FpGmNj-uqs^hQsNix{ryq$G#DO%WzU`o3DH4Sm*YBzC%Mu|{LFm+{H(mu!6V{gUR& znJbwe`1T3kU9di`xwl_ycUOBhutj~j(Uy?XH4PWDY)l0Hfo?2voSw6<=$GGDwIT8L z`SB;%8iRQ{k>{MmN>}?FYON(cTQBVWqPMjN;M)xCxaKSLDPOCMx8>w_LEhvPLT;v| zB7~paNUpl-m0F&?LgqriZ?UPD41RGCN)dMC_xe}yEb)AXq~D{H!wu4jRv;9T^lIf% z2ApJo#3NsoVx!J)BgOq1zB7fv;2o{S&njG()76VKFk_z5!z9&H)uFdk5SSK&{Sld4L_zD<&F17`zYVMrUAo`$U7u z9gE{vjK5TNoRXd7T_hwYOezUTvgrp5Akqun)PXpWL!_+jCo8bJZ4} zcrwfR!)Q1iit}kWrHNw`ZXHHI{5>?~k2X$DjE)^Fj5%)sOpWgM4}SFWrwVaimEU(s zcyJXLzmLJs(>U0h#fKNq9PbL}l09$6U!IYr+eRL!#q}XP^Y?_hcP#b(Bi{^8l)MMb zGrE+22A>;tE%j%k$-jh8*N2z7jL823c)_uy{5ieE%Y(_w+*ly#&TL23~^yJ0pbyz2pPrD_w5ba9%iuKPeo;mxOrb)QpJJ~+#a@4kz5Wz?{VDeP zQ|$Gp*y~TR*PmjqKgC{uioO06d;MVyn593(UVnrb)QpJJ~+ z#a@4kz5Wz?{VDePQ|$Gp*y~TR*PmjqKgC{uioO06d;Q_d4zu*9*y~TR*PmjqKgC{u zioO06d;KZ)`cv%nr`YRHvDcqsuRq0%@4WC|pq<K@I)W=qEU$MgtD83y$teEn=aC3YXZjR5w&GA|IG~$03z6DfK{7&$B#kYW~ zirc|8#kYg&ir0XzC|(Wb7b{tw{|V`Jc67TQ#W#S5gk%0Q!ZH85gk%1*!henUZtCfF zlyHoH zNcg7^zJE=(ODnzsJS_a1kl&c?bC2wIOhL~aLhly%&YT1ARPUtg`@vr<3B6B z8|g2bM1Bnyg=77k6pr;%QgYVMImN7>s$$m9Wy5IVKL0Vl1Hv)CXfm_Px6Am?8~?|Q z|Dy2cP`=I`-R`8~4)7`AvygWo^0M&n(Eo=q9ti)p;3e5^*AR~SxFQ_&!5%Celz$%j z7iN5SBEEBq8Gl9jzZL$^3&;Fa4cCO3pQXr8558o`^8X$9=46o@P<$gerI_*|#gwNN zQ$DPi@{D52cPXYktC;d}#gyk2Q+`Y_Hm^q`oFB0{u_$v zzoRGUhvlLFKH+WPrC;xMDd8Ulcg=LVVa0vmtm50jMa7h#6OQ9YRXC0x7lq^aaY^{= zNbi;^>?Z_^G%Pe8u?Z4<)d^|8Mxe_hPpjP`n(R5{~f?3Ezu(Wyz};pM)O- zFTaHGN%%*=cfZ!{vWk1btBOKG;B^=Yw3)d08=cAqOr11B^-7tPi!u&Clj!$*EQ^L1^mzCPw zIU}zK4?@21i*2qh{4jXY@3*-G8(Pe72E6o+B`zhLCEv2drG;botZ*Lk8;>n<$Bg`> z@XtWr@jAv=;a>sY@cAXKthgIo5&j*>yFa_cT@sG^Y6wStT@jA@>c9`8F#o>&LMa2n z-#~orOR+y@xF{UsKP4RFFAK-`E5ffcf0IjGRk$1TWcTyaMn_lj8Y5Yh4^ZO&@bDeHn_z3v6a;G~c{4-$pRg5Q!yTK=o|B`TwuO>`8pa;fq zAAazK>HRJF_mRHv1DGe4Ry*A;;itjv-$43?%feA#=Y^xbs=`rU7lnTU@pXN()76E4 zoAK8=T?c-MfcYWs1t-Gqz_{H0D#lmEX>eBfLzG`a`4w*mmxRAc`D>l7YUG!NWB&PL zCrm%)ze70YKQH`U#JBWmv~OYl&`!sLOI%Sn0e3yV#GMj;0K9Am+PC7R;B&$|AYZ%- z?OiwrUUJ7$R}-EB_axXK?hpC<1Mpq62e4nPxE;L9a8Wpx z|CDene_1$|zas3%gUfBMrWurhiKKmk_>V@nUyQIF6s^g0(zC{w?q#{O4(xg#Q!x zeYYV0ir)d|5BD&?eHh1=^mVxb;h4X)aLnJZaLivu_-=&1Bk6Lx6yE^O3O@F0JJBKdhMkGm0tSrI_-pV#>!AQ=V5$ z`7ywSrUQtZ>dBv1hh2whZl47nu>cZ65a_FmJ#Fbk^jC7#slGD*gZ>j zp#KYh5Zv?OE>{!IlXs#23;#TL)n4>}#W#R2EA9q2g#R1(-;nKc$%>#K^3C7@#W#Xe z!i!)Rb&aDv3ip5)<`JLp`@nrO7(ea`@sS<4Lowxv;xu?zG3C32UqblqV_j}saVPkg z@K+(fq1fe0!b@Q%E%_A26XAYv@5wHAPVvp)itxjb-^k6wyF-3=l0T2~3r~V?{Vd8a z{Nv!RQz*ZY51a5Og}(^@%U{NPcu$C*cHFWG$}jv~@RIY0Pw{5(u;PcndBv2U5{~m# zML5n^=Y`{Zby0X9?6{>VtPg}gLB`9E%fdegUUtV~ms%O}8{;1qj`3%NWBkXA|B~^4 z%J@Gg?AM1?v~OX*KKusStKv~G{~ITczrT+BExCyLQTzxvCHyVOKU~9nBmCq$p#N{8 zy(n%6pA`NT$d{e$aHoWS3w*<8J6u^f?ysH`j{B<>CFlO?MaA4-tt;k!YR9`n{uv)R zCEV1na8tj+QNO!{Y3JU4s>5X!F9YX=zsvGq)3K=dW^h^f2H2hNdAY+?6er;G!sC$N ziI*8w#U0?9@Mj@!|9pq5tMJPZ{)+Hv$ZxM8e*?4I&oSXxzM^m}-$~)WL--}tY|#@7&z@pY^Y>AwSZ>aslMTj62wk{Qgm!q0-cj$ynP9-;gq@-H0IJ0={{ zJ1@-k)`9j`SIqWzS@`3O@6|5XHyH90+hOz-Q+zB36dV=uLS!+@x9=h;{4TIt8}dj0lqbUfHTV|1j2ssJQ}Ckibhuqco)zZ0VcEAk+_;e! zg}F|+qu$|4MqXC(WssjU@`~^p*wwu|P+rAd;JV?{?o)P{S!r$;!^k2pA0%sN91|C=Zt_t3t zE4~d}6h4FaI={ZqofPJ}ue$^LQNn);Uebg83&WR$30Z!5B`@vi1(Yq4`ZKaX}Zk~3H$eT_&@1VU2b3ccNDK9GS0$&pL`#;}C`!nJ3 zWluk!{5{fJj{l20FTCk4a7Vj4CcFdOfy0sp?vn5);lDj;cUKH2 z4~6o@@oq>sjz4MPINl8l{~qG&*@5;U{2#$9arm~Vco(>&cmP~cO#hdJWBLu@nEn;v zn11rG$**wqpB9e(hlM|Yeaxa%$lJhu!fn`xY#(ZO!$zJLj^#Zm9P?iij`=?&?B8GFFmpxt8N}a{Z+GW~xv%=p z@pe~LybOF%_zRHVvKRUm_WSWO?e4Ph??T>v4DCblZQ!1CC?Dk$;6!l>oL0ORJT4r| zR}_xrJ1HE?R}#Jt`=pD?$e*y^e>>Oih8~?=-x=W;-!9=8U)J!r;i7Q-zR@YgJilL7 z%=7yd#XP@%Q8CZ&*A(;n0GAXqzPe(@*HFy(t|(@F?y*qbn7=;7j4x5l_)>})->_oF zmr>08?NZG6vWgjBUNPf4rkL>+6*In5ioO06d;KZ)`cv%nr`YRHvDcqsuRq0Je~P{S zwix{>_WD!o^{3eDPqEjZVy{2NUVnrb)QpJJ~+#a@4kz5Wz? z{VDePQ|$Gp*z50cqd&!7e~P{S6np(C_WD!o^{3eDPqEjZVy{2NUVnrb)QpJJ~+#a@4kz5Wz?{cSb+Q|$Gp*y~TR*PmjqKgC{uioO06d;KZ)`cv%n zC!Af0t%UQKpB49j&k6rB#6maEEZTH+vM*-Wm|TYZdm(@t*_lQhXD5T-cva`3CG2;ZgX% z=@RS}#ogeO!i=v6;Y-5I&vNAFlyI~M%fdef|4SQ~pM{(I!-~28S~q-IvELt7O#dD4 z5Bg~C4-3cr+eA3(XFxIalTzUs|BzzFpE0~kG5u#1(|=w#re6|{`Z#6eW#MnGg1w3_ zOPw?F^C~>;>8fJp|Dy2!gYe7$675UL>A$X+{x2)0{EA}A-Ehz+ZlM{|@X` z#ogeZCqwx@0QvGO?QTH$%i#NaI^2-r9&lPXwx40e)K^9^^|?zi< ziYYHDru?L0%1eqVKc$%Rvammo;P5_J$?5;RV*0NNQyTgclEBeD1^hEyY9NV~XDg zt|+Gbl5n)o8^Y0^z9Jm$>Er{Ue9`}qaP*%Rj{b**k05`GPN97ZzYKooN$f8uUIsoU z?BDl)7VTT{&ESgge?a(D3GJmw$8H-IlI?glr6x59t> zr!XH0GrgM;zk52ApL`>@L-@n+zwnFbU&2v8Y2m1!Vd1Esyzzh1_%9j%W#O>jit`P^ zVZXJ*)fJE4gZHVz`;fmS19*S>Ovpd^A@Gp!mndI@^8t#V1m}etl&5gs!pJWQ$MV*N zV|gzN$MSY;o85jA;pl%rIQq{R|KrBLdy{LCTQo=dHMrcQ+|MIEUTJ)8bZva}*J8Q9 z1NR#p>Rqlx?!SgR{of&s?Pq^|6@TfzR5-e0o7cN{h*-)j>wEX!PFa>@{D52`8yW87|e%P?88gdMtMfD53ksVSM0-!BAVe9`|yf= zc*QU`XA_; z%{cn#FUt9dqh?KJ(4OuVGrnDh`Mq6UG5nA;crwEm6*GKQu@5Q@E9S3f89d=4Q#Aw2 zNr4%Z-;?7N{quWZnzJ{}W`3_q%lW+_&HSE^<}&xoamD!gy%5d(o`hz8??5xZ2cVhf z@*}gn1Mh6HZx8QkG0V$yz`U3~%gb|+ykdTMZc#JOC2GFBs>Rel&*AZk;dyRNGtZrA z=DD)StQwvZi_HA;T$g5^+tSQ)S(G+jeV!}gMW&qR zig-mi&)q~`iU*#Xh|K)*Tm&!rXZk$1z$?bba|)4t{uKNC;XQklGku;Dh)g-(^GBwf z@9iT~&iC$-Dd&6i$dvOvcx1}?UOO`7d@miDa=vGdOgY~ZN2Z+bg(I{4e2=S{?^S0p zCnQ|d2mSNCBd^GO51^U*@|qLudu!%Cw&wBlY#w@aHWw9h=;XdCFQ)H@#>YH*rkwkl zyrP`@j*)$M#XkI&+47WP%8QD9c*Q>a<0iafA6~H!uh@s*YQig~ock=iSRS@_?i27L z`}X{lN6(A#NV}UC~9Ule0*Io_0vI5xEPlD#Zt##Wa&Zn$CCEu3B69>a;@l;O1DXkSMCWsN*r!)e3OK5mxZ$n%DahD(OahAW1vhHHlFh8u=`Ls#YPVa63FhEs;qhH(^Q zR`{&pyy2qZlHs!9is7o^n&G15zrd%di2*uaN2OjaMo~Kudy096cvrUWVmd&Vz_F!X1H#+VK{C% z#QgNg{1`bgoHCp?oH3j=oHtxFTrylXTrpfVTr*rZ+%W8ocGN>GZ(RRIPSpI*`oh+f zzoI;CIAb_#IB&RUxMa9&xMH|!xMsL+xMA2YqILOcWGPMzs5GMqM?F`PA=H(WGaGF&!XF^Fzgq- zs=RSOCbsW{M!1rv45tle3}+4J4HpfU43`a83|9@;4A%`e40|JAm9K|Jzv9Gj%5d5+ zf3HG^&l=7fE*dTwE*q{Gt{Scxt{ZL`jvKGBzI(U`!xcF(oHCp?oH3j=oHtxFTrylX zTrpfVTr*rZ+%W7Hb*j8^f3R78*)NQoGMqM?F`PA=H(WGaGF&!XF^FzjUi zGnS9P!^SIeVmM_uZ8&2%YdCMXXt-p!Y`9{$YPe>&Zn$CC^~^4Bk75444zH*W{tlbw zwBd~5tl_-jqT!O^vf+y1s^Oa9y5WXlytJ8Bo*u)A;gsRD;f&#|;k@CZ;gaF9;fmp^ z;hN#P;f7(nq?=uS!-?UP;k4n5;rPBFwx7I_7Y&yTmkn18R}I$;*9|ueJ9&Q*^VefI zF`P1-Hk>h>HJmqGG+Z)VHe4}WHC!`XH{3An&Zn$CC$@|D={TogUrwpeJ zXAEZz=M5JPmkgH;R}5DT*9_MUHw-&@|Jp3SVg4?luHTg5wBd~5tl_-jqGA4ypiZxB zxMH|!xMsL+xM7&TL#Wg1F`O7q8BQC{7|t5b8!j3y87>>H7_J(w8Lk^{7^ zFzn>KNX%c4;lyytaN2OjaMp0%aM5tdaM^IhaMf_laNTgjFn_0UR{J;1-)YqHl;N~t z{$7aopEaB}Tr^xVTsB-WTs2%XTsPb>?BqO9v;GYyhEs;qhBJnKhO>tAhKq(vhRcR4hO35ahUCx-buj@mze$5C^}aMp0%aM5tdaM^IhaMf_l zaNTgju)Ak={`otTx;*@yNzEz4{M|_{<`c=J%zw|DxfN;j-b1;i}=9;kx05VYhO2 zetHZihU59r#R#?lpR|!@4Dt=#4vv+ zQ~OUF=I>@|dDd{=aM5tdaM^IhaMf_laNTgjuzUCH{Pq}549D}nu|CsAo-v#?oHtxF zTrylXTrpfVTr*rZ+%W70O!*BbhEs;qhBJoadFZIGypb0TmkgH;R}5DT*9_MUHw?Sg zru>Ez!~ES(Ua>rB!x_Wze0TJpH}az4lHs!9is7o^n&GOP@w|F0 zU&hF@hVzDthD(OahAW1vhHHlFh8u?6drbKaCx-buro3W#(}wvwrCOdfoHtxFTrylX zTrpfVTr*rZ+%Oyu^v3e^teKtv#Bj=R+Hl5j)^Of%(QwId*>J^h)o{&l-EhOOTRXeF zJ%$s*DZ^>Q8N*q_dBa7+CBtRI6~k4-HN$no4a07oDZk;waLRDnFn{-yS8PvN!+FC+ z!zIII!xh6-!!^To!wtjyol~9P9>a;@l;O1DjNz=|yy2qZlHs!9is7o^n&G>H7_J(w8Lk^{7Dl)#Bj=R z+Hl5j)^Of%(QwId*>J^h)o{&l-EhOOTW`v5IDWq>)?doV(}pvKvxf7Ai-t>v%Z4k4 ztA=Za>xLVK-3C*B!-?UP;rRWrsGp3HXAS2K7Y&yTmkn18R}I$;*9|ueyN#y&h7-g5 z9aLViylKN3!&$?5!$rd-!)3!2!&SpI!*#BwB@`~k445tjI4fA(ab@;5| zyy2qZlHs!9is7o^n&Gv%Z4k4tA=Za>xLVK z-Fr>>4JU?EhSP>KhO>tAhKq(vhRcR4hO35ahU?A|v!|2>8i!zsgQ z!x_U_!+FC+!zIII!xh6-!!^To!wtjyon2jj{9RqmiQ$ytv|;|Pt`46y%-_+~@}l9A z;j-b1;i}=9;kx05VfWDN{PY-(-_MWZTgu4OhBJn0U(FgbP9t(zG;HsjVE8k=1A*p4UG5ehT8nSxt)AeS$=BOCKm1@CiY z-=W-#1(%4*~gia$k2c6W^#LQKYqa94^8dO&E%RHaZf+FW$U)*wmK%Y$>k4??c3xI=4Lj@ ziaO}=?f(530xr5r;4j_0z`LF|vf$csT!Kfs@vg752KCJP)7M7;Zx%lOeq9XH5g+7! z1ouvS(!%j}?!OOD@q5@e09no$zDGR%H114)p>Vui*%qE;+cG-){?!CL?8S%Stqupa z2VYEAU-YZ5I{{Y> (i % 8) } - for name, v := range map[string]any{ - "pool_base": binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()), - "pool_mask": uint32(1)<<(32-bpfPool.Bits()) - 1, - "pref6": b.prefix.Masked().Addr().As16(), - "mask6": mask, - "v4pos": v4Positions(b.prefix.Bits()), - "slow_mark": uint32(bpfSlowMark), - } { - spec.Variables[name].Set(v) // the program declares each, of this size - } - objs := new(nat64Objects) - if err := spec.LoadAndAssign(objs, nil); err != nil { + pref, pos := b.prefix.Masked().Addr().As16(), v4Positions(b.prefix.Bits()) + set(nat64VarPoolBase, nativeEndian.AppendUint32(nil, binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()))) + set(nat64VarPoolMask, nativeEndian.AppendUint32(nil, uint32(1)<<(32-bpfPool.Bits())-1)) + set(nat64VarPref6, pref[:]) + set(nat64VarMask6, mask[:]) + set(nat64VarV4pos, pos[:]) + set(nat64VarSlowMark, nativeEndian.AppendUint32(nil, bpfSlowMark)) + objs, err := bpfLoad(bpfProgram{code: nat64Code, maps: nat64Maps, relocs: nat64Relocs, rodata: ro, license: nat64License}) + if err != nil { return fmt.Errorf("loading the BPF translator: %w", err) } b.objs = objs - l, err := link.AttachTCX(link.TCXOptions{Interface: b.ifindex, Program: objs.Nat64, Attach: ebpf.AttachTCXEgress}) + fd, err := objs.attachTCX(b.ifindex) if err != nil { return fmt.Errorf("attaching the BPF translator, which needs Linux 6.6 or later: %w", err) } - b.link = l + b.link = os.NewFile(uintptr(fd), "tcx") return nil } @@ -210,24 +203,22 @@ func (b *bpfNAT64) sweep() { if b.idle == nil { b.idle = map[[4]byte]int{} } - var a4 [4]byte - var a6 [16]byte - var gone [][4]byte - it := b.objs.V6Of.Iterate() - for it.Next(&a4, &a6) { + v4of, v6of := b.objs.maps[nat64MapV4Of], b.objs.maps[nat64MapV6Of] + for _, key := range v6of.keys() { + a4 := [4]byte(key) if active[netip.AddrFrom4(a4)] { delete(b.idle, a4) continue } if b.idle[a4]++; b.idle[a4] >= bpfIdleSweeps { - gone = append(gone, a4) - b.objs.V4Of.Delete(a6) + a6 := make([]byte, 16) + if v6of.lookup(key, a6) == nil { + v4of.delete(a6) + } + v6of.delete(key) + delete(b.idle, a4) } } - for _, a := range gone { - b.objs.V6Of.Delete(a) - delete(b.idle, a) - } } // conntrackSources4 dumps conntrack's IPv4 flows and returns the original sources within pool. @@ -305,15 +296,16 @@ func route4MTU(ifi int, dst netip.Prefix, mtu int) error { // slow path dropped. func (b *bpfNAT64) drops() string { var parts []string + stats := b.objs.maps[nat64MapStats] + per := make([]byte, stats.valueSize) for i, name := range bpfStatNames { - var per []uint64 if name == "" { continue } - b.objs.Stats.Lookup(uint32(i), &per) // every index of an array is there + stats.lookup(nativeEndian.AppendUint32(nil, uint32(i)), per) // every index of an array is there var n uint64 - for _, v := range per { - n += v + for c := 0; c < len(per); c += 8 { + n += nativeEndian.Uint64(per[c:]) } if n != 0 { parts = append(parts, fmt.Sprintf("%s %d", name, n)) diff --git a/nat64bpf_integration_test.go b/nat64bpf_integration_test.go index 60f324d..d55ab33 100644 --- a/nat64bpf_integration_test.go +++ b/nat64bpf_integration_test.go @@ -365,15 +365,7 @@ func bpfEndToEnd(t *testing.T, prefix netip.Prefix) { noDrops("after the errors") // The mappings stay while conntrack has their flows, and go two sweeps after it forgets them - mappings := func() int { - n := 0 - var a4 [4]byte - var a6 [16]byte - for it := b.objs.V6Of.Iterate(); it.Next(&a4, &a6); { - n++ - } - return n - } + mappings := func() int { return len(b.objs.maps[nat64MapV6Of].keys()) } before := mappings() b.sweep() b.sweep() diff --git a/nat64slow.go b/nat64slow.go index a58d8d3..e7dfe33 100644 --- a/nat64slow.go +++ b/nat64slow.go @@ -13,8 +13,6 @@ import ( "sync/atomic" "time" - "github.com/cilium/ebpf" - "github.com/cilium/ebpf/perf" "golang.org/x/sys/unix" ) @@ -34,8 +32,8 @@ const ( type slowPath struct { prefix netip.Prefix - v4of *ebpf.Map - v6of *ebpf.Map + v4of *bpfMap + v6of *bpfMap dev *net.Interface mtu6 int // what an IPv6 packet sent on may be at most fd int // a packet socket, sending marked @@ -59,13 +57,13 @@ type slowGroup struct { expires time.Time } -func newSlowPath(prefix netip.Prefix, objs *nat64Objects, dev *net.Interface, mtu6 int) (*slowPath, error) { +func newSlowPath(prefix netip.Prefix, v4of, v6of *bpfMap, dev *net.Interface, mtu6 int) (*slowPath, error) { fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0) if err != nil { return nil, fmt.Errorf("packet socket: %w", err) } unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_MARK, bpfSlowMark) // CAP_NET_RAW, which the socket took, allows it - return &slowPath{prefix: prefix, v4of: objs.V4Of, v6of: objs.V6Of, dev: dev, mtu6: min(mtu6, dev.MTU), fd: fd, groups: map[slowKey]*slowGroup{}, now: time.Now}, nil + return &slowPath{prefix: prefix, v4of: v4of, v6of: v6of, dev: dev, mtu6: min(mtu6, dev.MTU), fd: fd, groups: map[slowKey]*slowGroup{}, now: time.Now}, nil } func (p *slowPath) close() { unix.Close(p.fd) } @@ -82,37 +80,23 @@ func (p *slowPath) send(pkt []byte) error { } // serve reads what the program hands over until the reader is closed. -func (p *slowPath) serve(rd *perf.Reader) { +func (p *slowPath) serve(rd *ringReader) { for { - rec, err := rd.Read() + rec, err := rd.read() if err != nil { return } - p.record(rec) + p.handle(rec) } } -// record takes one record of the perf buffer: a sample, or the count of those it had no room for. -func (p *slowPath) record(rec perf.Record) { - if rec.LostSamples > 0 { - p.dropped.Add(rec.LostSamples) - return - } - p.handle(rec.RawSample) -} - -// handle takes one sample: the frame's length, then the frame. -func (p *slowPath) handle(sample []byte) { - if len(sample) < 4 { - p.dropped.Add(1) - return - } - n := int(binary.NativeEndian.Uint32(sample)) - if n < 14 || len(sample) < 4+n { +// handle takes one frame the program handed over. +func (p *slowPath) handle(frame []byte) { + if len(frame) < 14 { p.dropped.Add(1) return } - if err := p.packet(sample[4+14 : 4+n]); err != nil { + if err := p.packet(frame[14:]); err != nil { p.dropped.Add(1) debugf("[nat64] slow path: %v", err) } @@ -227,7 +211,7 @@ func (p *slowPath) whole(hdr []byte, proto uint8, payload []byte) error { // to6 translates IPv4 from the pool's side: ICMP echo, or UDP whose checksum it computes. func (p *slowPath) to6(hdr []byte, proto uint8, payload []byte) error { var dst [16]byte - if err := p.v6of.Lookup([4]byte(hdr[16:20]), &dst); err != nil { + if err := p.v6of.lookup(hdr[16:20], dst[:]); err != nil { return fmt.Errorf("no source has %s", netip.AddrFrom4([4]byte(hdr[16:20]))) } src := embed4(p.prefix, [4]byte(hdr[12:16])) @@ -387,9 +371,9 @@ func l4Checksum(src, dst []byte, proto uint8, msg []byte) uint16 { } // map4 is map4 of bpf/nat64.c: the pool address of an IPv6 source, a free one the first time. -func map4(v4of, v6of *ebpf.Map, src [16]byte) ([4]byte, bool) { +func map4(v4of, v6of *bpfMap, src [16]byte) ([4]byte, bool) { var a [4]byte - if v4of.Lookup(src, &a) == nil { + if v4of.lookup(src[:], a[:]) == nil { return a, true } h := hash6(src) @@ -397,14 +381,14 @@ func map4(v4of, v6of *ebpf.Map, src [16]byte) ([4]byte, bool) { mask := uint32(1)<<(32-bpfPool.Bits()) - 1 for i := range uint32(bpfProbes) { binary.BigEndian.PutUint32(a[:], base+((h+i)&mask)) - if v6of.Update(a, src, ebpf.UpdateNoExist) != nil { + if v6of.update(a[:], src[:], unix.BPF_NOEXIST) != nil { continue } - if v4of.Update(src, a, ebpf.UpdateNoExist) == nil { + if v4of.update(src[:], a[:], unix.BPF_NOEXIST) == nil { return a, true } - v6of.Delete(a) // the program mapped the same source meanwhile - return a, v4of.Lookup(src, &a) == nil + v6of.delete(a[:]) // the program mapped the same source meanwhile + return a, v4of.lookup(src[:], a[:]) == nil } return a, false } diff --git a/nat64slow_integration_test.go b/nat64slow_integration_test.go index a80a234..2d3253f 100644 --- a/nat64slow_integration_test.go +++ b/nat64slow_integration_test.go @@ -13,8 +13,8 @@ import ( "strings" "syscall" "testing" + "unsafe" - "github.com/cilium/ebpf" "golang.org/x/sys/unix" ) @@ -36,17 +36,8 @@ func newSlowFixture(t *testing.T, mtu int) *slowFixture { t.Fatal(err) } ifi.MTU = mtu - v4of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 16, ValueSize: 4, MaxEntries: 64}) - if err != nil { - t.Fatal(err) - } - v6of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 4, ValueSize: 16, MaxEntries: 64}) - if err != nil { - t.Fatal(err) - } - objs := &nat64Objects{nat64Maps: nat64Maps{V4Of: v4of, V6Of: v6of}} - t.Cleanup(func() { objs.Close() }) - p, err := newSlowPath(nat64WKP, objs, ifi, 1500) + v4of, v6of := testMaps(t, 64, 64) + p, err := newSlowPath(nat64WKP, v4of, v6of, ifi, 1500) if err != nil { t.Fatal(err) } @@ -145,7 +136,7 @@ func TestSlowPathTranslates(t *testing.T) { p := f.p host := netip.MustParseAddr("2001:db8::5").As16() pool := [4]byte{198, 18, 0, 5} - if err := p.v6of.Put(pool, host); err != nil { + if err := p.v6of.update(pool[:], host[:], 0); err != nil { t.Fatal(err) } @@ -216,7 +207,7 @@ func TestSlowPathTranslates(t *testing.T) { } hdr, msg = reassembled(t, f.sent(), 600) var a4 [4]byte - if err := p.v4of.Lookup(src.As16(), &a4); err != nil { + if err := p.v4of.lookup(src.AsSlice(), a4[:]); err != nil { t.Fatal(err) } base, mask := binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()), uint32(1)<<(32-bpfPool.Bits())-1 @@ -245,7 +236,7 @@ func TestSlowPathTranslates(t *testing.T) { for i := range uint32(bpfProbes) { var slot [4]byte binary.BigEndian.PutUint32(slot[:], base+((hash6(busy)+i)&mask)) - p.v6of.Put(slot, host) + p.v6of.update(slot[:], host[:], 0) } if err := p.packet(v6Packet("2001:db8::99", "64:ff9b::1", unix.IPPROTO_ICMPV6, reply[:8])); err == nil || !strings.Contains(err.Error(), "full") { t.Fatalf("want the pool full, got %v", err) @@ -274,24 +265,13 @@ func TestMap4WithoutRoom(t *testing.T) { if !ownNetns(t) { return } - v4of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 16, ValueSize: 4, MaxEntries: 1}) - if err != nil { - t.Fatal(err) - } - defer v4of.Close() - v6of, err := ebpf.NewMap(&ebpf.MapSpec{Type: ebpf.Hash, KeySize: 4, ValueSize: 16, MaxEntries: 4}) - if err != nil { - t.Fatal(err) - } - defer v6of.Close() - v4of.Put(netip.MustParseAddr("2001:db8::1").As16(), [4]byte{198, 18, 0, 1}) + v4of, v6of := testMaps(t, 1, 4) + v4of.update(netip.MustParseAddr("2001:db8::1").AsSlice(), []byte{198, 18, 0, 1}, 0) if _, ok := map4(v4of, v6of, netip.MustParseAddr("2001:db8::2").As16()); ok { t.Fatal("a full map should refuse a new source") } - var a [4]byte - var s [16]byte - if v6of.Iterate().Next(&a, &s) { - t.Fatalf("the claimed slot %v should be given back", a) + if keys := v6of.keys(); len(keys) != 0 { + t.Fatalf("the claimed slot %v should be given back", keys) } } @@ -317,12 +297,11 @@ func TestBPFProgram(t *testing.T) { } src := netip.MustParseAddr("2001:db8::abcd") udp := []byte{0, 1, 0, 2, 0, 8, 0x12, 0x34} - ret, err := b.objs.Nat64.Run(&ebpf.RunOptions{Data: frame(v6Packet(src.String(), "64:ff9b::c000:202", unix.IPPROTO_UDP, udp), unix.ETH_P_IPV6)}) - if err != nil || ret != 7 { // TC_ACT_REDIRECT - t.Fatalf("want the packet redirected, got %d: %v", ret, err) + if ret, _, _ := b.objs.testRun(t, frame(v6Packet(src.String(), "64:ff9b::c000:202", unix.IPPROTO_UDP, udp), unix.ETH_P_IPV6), nil); ret != 7 { // TC_ACT_REDIRECT + t.Fatalf("want the packet redirected, got %d", ret) } var a [4]byte - if err := b.objs.V4Of.Lookup(src.As16(), &a); err != nil { + if err := b.objs.maps[nat64MapV4Of].lookup(src.AsSlice(), a[:]); err != nil { t.Fatal(err) } base, mask := binary.BigEndian.Uint32(bpfPool.Addr().AsSlice()), uint32(1)<<(32-bpfPool.Bits())-1 @@ -338,11 +317,10 @@ func TestBPFProgram(t *testing.T) { gone.stop() // struct __sk_buff from its start to mark, the third word - ctx := []uint32{0, 0, bpfSlowMark} - out := make([]uint32, 48) // the whole struct __sk_buff - ret, err = b.objs.Nat64.Run(&ebpf.RunOptions{Data: frame(make([]byte, 40), unix.ETH_P_IPV6), Context: ctx, ContextOut: out}) - if err != nil || ret != 7 || out[2] != 0 { - t.Fatalf("want a marked packet redirected with its mark cleared, got %d, mark %#x: %v", ret, out[2], err) + ctx := nativeEndian.AppendUint32(make([]byte, 8), bpfSlowMark) + ret, _, out := b.objs.testRun(t, frame(make([]byte, 40), unix.ETH_P_IPV6), ctx) + if mark := nativeEndian.Uint32(out[8:]); ret != 7 || mark != 0 { + t.Fatalf("want a marked packet redirected with its mark cleared, got %d, mark %#x", ret, mark) } } @@ -395,7 +373,7 @@ func TestBPFNAT64WithoutPrivileges(t *testing.T) { t.Fatal(err) } defer b.stop() - b.objs.V6Of.Put([4]byte{198, 18, 0, 1}, netip.MustParseAddr("2001:db8::1").As16()) + b.objs.maps[nat64MapV6Of].update([]byte{198, 18, 0, 1}, netip.MustParseAddr("2001:db8::1").AsSlice(), 0) if err := syscall.Setresuid(-1, 65534, -1); err != nil { t.Fatal(err) } @@ -409,9 +387,7 @@ func TestBPFNAT64WithoutPrivileges(t *testing.T) { t.Fatalf("want the old device's removal refused, got %v", addErr) } joolWithFDs(t, 0, func() error { b.sweep(); b.sweep(); return nil }) - var a [4]byte - var s [16]byte - if !b.objs.V6Of.Iterate().Next(&a, &s) { + if len(b.objs.maps[nat64MapV6Of].keys()) == 0 { t.Fatal("a sweep that cannot ask conntrack should remove nothing") } // The slow path's drops are reported with the program's @@ -455,7 +431,7 @@ func TestSlowPathWithoutDescriptors(t *testing.T) { if err != nil { t.Fatal(err) } - if err := joolWithFDs(t, 0, func() error { _, err := newSlowPath(nat64WKP, &nat64Objects{}, lo, 1500); return err }); !errors.Is(err, unix.EMFILE) { + if err := joolWithFDs(t, 0, func() error { _, err := newSlowPath(nat64WKP, nil, nil, lo, 1500); return err }); !errors.Is(err, unix.EMFILE) { t.Fatalf("want the socket refused, got %v", err) } } @@ -477,18 +453,12 @@ func TestBPFICMPTranslation(t *testing.T) { } defer b.stop() host := netip.MustParseAddr("2001:db8::5") - b.objs.V6Of.Put([4]byte{198, 18, 0, 5}, host.As16()) - b.objs.V4Of.Put(host.As16(), [4]byte{198, 18, 0, 5}) + b.objs.maps[nat64MapV6Of].update([]byte{198, 18, 0, 5}, host.AsSlice(), 0) + b.objs.maps[nat64MapV4Of].update(host.AsSlice(), []byte{198, 18, 0, 5}, 0) run := func(pkt []byte, proto uint16) ([]byte, bool) { t.Helper() - out := make([]byte, 2000) - frame := append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...) - opts := &ebpf.RunOptions{Data: frame, DataOut: out} - ret, err := b.objs.Nat64.Run(opts) - if err != nil { - t.Fatal(err) - } - return opts.DataOut[14:], ret == 7 + ret, out, _ := b.objs.testRun(t, append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...), nil) + return out[14:], ret == 7 } udp := []byte{0, 7, 0x30, 0x39, 0, 8, 0xab, 0xcd} @@ -632,15 +602,10 @@ func TestBPFRFC6052(t *testing.T) { if err := b.load(); err != nil { t.Fatal(err) } - b.objs.V6Of.Put([4]byte{198, 18, 0, 5}, host.As16()) + b.objs.maps[nat64MapV6Of].update([]byte{198, 18, 0, 5}, host.AsSlice(), 0) run := func(pkt []byte, proto uint16) ([]byte, bool) { - out := make([]byte, 200) - opts := &ebpf.RunOptions{Data: append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...), DataOut: out} - ret, err := b.objs.Nat64.Run(opts) - if err != nil { - t.Fatal(err) - } - return opts.DataOut[14:], ret == 7 + ret, out, _ := b.objs.testRun(t, append(binary.BigEndian.AppendUint16(make([]byte, 12), proto), pkt...), nil) + return out[14:], ret == 7 } got, ok := run(v4Packet("192.0.2.33", "198.18.0.5", unix.IPPROTO_UDP, 0, udp), unix.ETH_P_IP) if src := netip.AddrFrom16([16]byte(got[8:24])); !ok || src != netip.MustParseAddr(want) { @@ -656,3 +621,43 @@ func TestBPFRFC6052(t *testing.T) { b.stop() } } + +// testMaps creates the two maps of the address mapping, with room for so many sources and pool +// addresses. +func testMaps(t *testing.T, sources, pool uint32) (v4of, v6of *bpfMap) { + t.Helper() + v4of, err := bpfMapCreate(bpfMapSpec{name: "v4_of", typ: unix.BPF_MAP_TYPE_HASH, keySize: 16, valueSize: 4, maxEntries: sources}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(v4of.close) + v6of, err = bpfMapCreate(bpfMapSpec{name: "v6_of", typ: unix.BPF_MAP_TYPE_HASH, keySize: 4, valueSize: 16, maxEntries: pool}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(v6of.close) + return v4of, v6of +} + +// testRun runs the program once on a frame (BPF_PROG_TEST_RUN), with ctx as the start of its +// struct __sk_buff, and returns the verdict, the frame and the whole context after. +func (o *bpfObjects) testRun(t *testing.T, frame, ctx []byte) (uint32, []byte, []byte) { + t.Helper() + out, ctxOut := make([]byte, 2000), make([]byte, 192) + attr := struct { + prog, retval, dataSizeIn, dataSizeOut uint32 + dataIn, dataOut uint64 + repeat, duration uint32 + ctxSizeIn, ctxSizeOut uint32 + ctxIn, ctxOut uint64 + }{prog: uint32(o.prog), dataSizeIn: uint32(len(frame)), dataSizeOut: uint32(len(out)), dataIn: ptr(frame), dataOut: ptr(out)} + if ctx != nil { + attr.ctxSizeIn, attr.ctxSizeOut, attr.ctxIn, attr.ctxOut = uint32(len(ctx)), uint32(len(ctxOut)), ptr(ctx), ptr(ctxOut) + } + if _, err := bpfSys(unix.BPF_PROG_TEST_RUN, unsafe.Pointer(&attr), unsafe.Sizeof(attr)); err != nil { + t.Fatal(err) + } + runtime.KeepAlive(frame) + runtime.KeepAlive(ctx) + return attr.retval, out[:attr.dataSizeOut], ctxOut +} diff --git a/nat64slow_test.go b/nat64slow_test.go index c735b82..67109c7 100644 --- a/nat64slow_test.go +++ b/nat64slow_test.go @@ -8,28 +8,16 @@ import ( "testing" "time" - "github.com/cilium/ebpf/perf" "github.com/mdlayher/netlink" ) -// A sample too short for its length or for an Ethernet frame is dropped, and so is a record of -// samples the buffer had no room for. -func TestSlowPathRecords(t *testing.T) { +// A frame too short for an Ethernet header, or with no IP after it, is dropped. +func TestSlowPathFrames(t *testing.T) { p := &slowPath{} - sample := func(n int, frame []byte) []byte { - return append(binary.NativeEndian.AppendUint32(nil, uint32(n)), frame...) - } - for _, rec := range []perf.Record{ - {RawSample: []byte{1, 2}}, - {RawSample: sample(10, make([]byte, 10))}, // shorter than an Ethernet header - {RawSample: sample(100, make([]byte, 20))}, // shorter than it says - {RawSample: sample(20, make([]byte, 20))}, // no IP after the Ethernet header - {LostSamples: 3}, - } { - p.record(rec) - } - if n := p.dropped.Load(); n != 7 { - t.Fatalf("want 4 bad samples and 3 lost counted, got %d", n) + p.handle(make([]byte, 10)) + p.handle(make([]byte, 20)) + if n := p.dropped.Load(); n != 2 { + t.Fatalf("want both frames dropped, got %d", n) } } diff --git a/tools/bpfgen/main.go b/tools/bpfgen/main.go new file mode 100644 index 0000000..4d228c2 --- /dev/null +++ b/tools/bpfgen/main.go @@ -0,0 +1,238 @@ +// Command bpfgen compiles a BPF program for either byte order and writes what sixup loads it with, +// so that sixup itself parses no ELF and no BTF: the instructions, the maps, the read-only data and +// the relocations that tie them together. It wants clang and the libbpf headers. +// +// Usage: go run ./tools/bpfgen bpf/nat64.c nat64 +// +// The program is one function in one section. Its maps are struct map_def values in the section +// maps, and its variables are volatile const ones in .rodata, which sixup fills in and freezes. +package main + +import ( + "bytes" + "debug/elf" + "encoding/binary" + "errors" + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" +) + +// The architectures Go builds for, by byte order +var orders = []struct { + target string + order binary.ByteOrder + arches string +}{ + {"bpfel", binary.LittleEndian, "386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64"}, + {"bpfeb", binary.BigEndian, "mips || mips64 || ppc64 || s390x"}, +} + +const rBPF6464 = 1 // R_BPF_64_64, the relocation of a 64-bit load, which debug/elf does not name + +type mapDef struct { + name string + typ, keySize, valueSize, max, flags uint32 +} + +type varDef struct { + name string + off, size uint64 +} + +type reloc struct { + insn, m int + off int64 // where in the read-only data, or -1 for the map itself +} + +type object struct { + code []byte + maps []mapDef // the read-only data last + rodata []byte + vars []varDef + relocs []reloc + license string +} + +func main() { + if len(os.Args) != 3 { + fmt.Fprintln(os.Stderr, "usage: bpfgen ") + os.Exit(2) + } + if err := run(os.Args[1], os.Args[2]); err != nil { + fmt.Fprintln(os.Stderr, "bpfgen:", err) + os.Exit(1) + } +} + +func run(src, ident string) error { + dir, err := os.MkdirTemp("", "bpfgen") + if err != nil { + return err + } + defer os.RemoveAll(dir) + var first *object + for _, o := range orders { + path := filepath.Join(dir, o.target+".o") + cmd := exec.Command("clang", "-O2", "-Wall", "-Werror", "-target", o.target, "-mcpu=v3", "-c", src, "-o", path) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("clang -target %s: %w", o.target, err) + } + obj, err := parse(path, o.order) + if err != nil { + return fmt.Errorf("%s: %w", o.target, err) + } + if first == nil { + first = obj + } else if !slices.Equal(first.maps, obj.maps) || !slices.Equal(first.vars, obj.vars) { + return errors.New("the byte orders disagree on the maps or the variables") + } + base := ident + "_" + o.target + if err := os.WriteFile(base+".bin", obj.code, 0o644); err != nil { + return err + } + if err := write(base+".go", perOrder(src, ident, base, o.arches, obj)); err != nil { + return err + } + } + return write(ident+"_bpf.go", shared(src, ident, first)) +} + +// parse reads the program, its maps, its read-only data and its relocations from an object. +func parse(path string, order binary.ByteOrder) (*object, error) { + f, err := elf.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var prog *elf.Section + section := map[string]elf.SectionIndex{} + for i, s := range f.Sections { + section[s.Name] = elf.SectionIndex(i) + if s.Flags&elf.SHF_EXECINSTR != 0 && s.Size > 0 { + if prog != nil { + return nil, fmt.Errorf("one program section expected, found %s and %s", prog.Name, s.Name) + } + prog = s + } + } + if prog == nil { + return nil, errors.New("no program") + } + obj := &object{} + if obj.code, err = prog.Data(); err != nil { + return nil, err + } + data := func(name string) []byte { + if s := f.Section(name); s != nil { + b, _ := s.Data() + return b + } + return nil + } + obj.license = strings.TrimRight(string(data("license")), "\x00") + obj.rodata = data(".rodata") + syms, err := f.Symbols() + if err != nil { + return nil, err + } + maps := data("maps") + var mapOffs []uint64 + for _, s := range syms { + switch { + case s.Section == section["maps"] && len(maps) > 0: + b := maps[s.Value:] + obj.maps = append(obj.maps, mapDef{s.Name, order.Uint32(b), order.Uint32(b[4:]), order.Uint32(b[8:]), order.Uint32(b[12:]), order.Uint32(b[16:])}) + mapOffs = append(mapOffs, s.Value) + case s.Section == section[".rodata"] && obj.rodata != nil && elf.ST_TYPE(s.Info) == elf.STT_OBJECT: + obj.vars = append(obj.vars, varDef{s.Name, s.Value, s.Size}) + } + } + slices.SortFunc(obj.vars, func(a, b varDef) int { return int(a.off) - int(b.off) }) + rodataMap := len(obj.maps) + obj.maps = append(obj.maps, mapDef{".rodata", 2, 4, uint32(len(obj.rodata)), 1, 0x80}) // BPF_MAP_TYPE_ARRAY, BPF_F_RDONLY_PROG + + rels := data(".rel" + prog.Name) + for b := rels; len(b) >= 16; b = b[16:] { + off, info := order.Uint64(b), order.Uint64(b[8:]) + if info&0xffffffff != rBPF6464 || info>>32 == 0 || int(info>>32) > len(syms) { + return nil, fmt.Errorf("unsupported relocation %#x at %#x", info, off) + } + s := syms[info>>32-1] // Symbols leaves out the null symbol + r := reloc{insn: int(off / 8), off: -1} + switch s.Section { + case section["maps"]: + r.m = slices.Index(mapOffs, s.Value) + case section[".rodata"]: + r.m, r.off = rodataMap, int64(s.Value)+int64(int32(order.Uint32(obj.code[off+4:]))) + default: + return nil, fmt.Errorf("relocation against %s, which is neither a map nor read-only data", s.Name) + } + obj.relocs = append(obj.relocs, r) + } + return obj, nil +} + +func header(src, tags string) *bytes.Buffer { + b := new(bytes.Buffer) + fmt.Fprintf(b, "// Code generated by tools/bpfgen from %s; DO NOT EDIT.\n\n//go:build %s\n\npackage main\n\n", src, tags) + return b +} + +// perOrder is what differs between the byte orders: the code, its relocations and the read-only +// data as compiled. +func perOrder(src, ident, base, arches string, obj *object) []byte { + b := header(src, "linux && ("+arches+")") + fmt.Fprintf(b, "import _ \"embed\"\n\n//go:embed %s.bin\nvar %sCode []byte\n\n", base, ident) + fmt.Fprintf(b, "var %sRodata = %#v\n\n", ident, obj.rodata) + fmt.Fprintf(b, "var %sRelocs = []bpfReloc{\n", ident) + for _, r := range obj.relocs { + fmt.Fprintf(b, "\t{insn: %d, m: %s, off: %d},\n", r.insn, mapConst(ident, obj.maps[r.m].name), r.off) + } + b.WriteString("}\n") + return b.Bytes() +} + +// shared is what both byte orders have alike: the maps, the variables and the license. +func shared(src, ident string, obj *object) []byte { + b := header(src, "linux") + fmt.Fprintf(b, "const %sLicense = %q\n\n", ident, obj.license) + fmt.Fprintf(b, "// The maps, the read-only data last\nconst (\n") + for i, m := range obj.maps { + fmt.Fprintf(b, "\t%s = %d\n", mapConst(ident, m.name), i) + } + fmt.Fprintf(b, ")\n\nvar %sMaps = []bpfMapSpec{\n", ident) + for _, m := range obj.maps { + fmt.Fprintf(b, "\t%s: {name: %q, typ: %d, keySize: %d, valueSize: %d, maxEntries: %d, flags: %#x},\n", mapConst(ident, m.name), m.name, m.typ, m.keySize, m.valueSize, m.max, m.flags) + } + fmt.Fprintf(b, "}\n\n// Where each variable sits in the read-only data, and its size\nvar (\n") + for _, v := range obj.vars { + fmt.Fprintf(b, "\t%sVar%s = bpfVar{off: %d, size: %d}\n", ident, camel(v.name), v.off, v.size) + } + b.WriteString(")\n") + return b.Bytes() +} + +func mapConst(ident, name string) string { return ident + "Map" + camel(name) } + +// camel turns a C name into the rest of a Go one: v4_of becomes V4Of, .rodata Rodata. +func camel(name string) string { + var b strings.Builder + for w := range strings.FieldsFuncSeq(name, func(r rune) bool { return r == '_' || r == '.' }) { + b.WriteString(strings.ToUpper(w[:1]) + w[1:]) + } + return b.String() +} + +func write(path string, src []byte) error { + out, err := format.Source(src) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + return os.WriteFile(path, out, 0o644) +}