diff --git a/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml new file mode 100644 index 00000000000000..be9bbcf473e850 --- /dev/null +++ b/Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/apple,mesa-fingerprint.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Mesa fingerprint sensor + +maintainers: + - Chromatischer + +description: | + Fingerprint sensor on an SPI bus of an Apple SoC, normally owned by the + Secure Enclave Processor. The chip-select setup and hold delays have to be + produced by the controller, so the device must use the native chip select of + its parent controller and not a GPIO one. + + The SPI mode is not given a default here because the boards that carry this + sensor have not been observed to agree on one, so each board description has + to state the mode it needs. + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + const: apple,mesa-fingerprint + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + description: Data-ready line from the sensor. + + enable-gpios: + maxItems: 1 + description: Power line to the sensor, asserted to power it up. + + firmware-name: + maxItems: 1 + description: + Calibration blob for this sensor. It is per-device factory data written + at manufacture, so it is neither redistributable nor derivable from the + compatible; the board description has to name it. + + spi-cpha: true + + spi-cpol: true + +required: + - compatible + - reg + - enable-gpios + - firmware-name + +unevaluatedProperties: false + +examples: + - | + #include + + spi { + #address-cells = <1>; + #size-cells = <0>; + + fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + firmware-name = "apple/mesacal-j313.bin"; + }; + }; diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index e179199dbd3b54..dc0287b59832b0 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -54,6 +54,15 @@ properties: power-domains: maxItems: 1 + apple,dma-range: + description: | + The DART's IOVA aperture as one 64-bit base address followed by one + 64-bit size. Present when the device only translates IOVAs inside a + limited range, for example the SEP which only accepts IOVAs below 4 GiB. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 4 + maxItems: 4 + required: - compatible - reg diff --git a/MAINTAINERS b/MAINTAINERS index 95e6cfee170837..f6fac4e4830c0e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2534,6 +2534,7 @@ F: Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml F: Documentation/devicetree/bindings/gpu/apple,agx.yaml F: Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml F: Documentation/devicetree/bindings/i2c/apple,i2c.yaml +F: Documentation/devicetree/bindings/input/apple,mesa-fingerprint.yaml F: Documentation/devicetree/bindings/input/touchscreen/apple,z2-multitouch.yaml F: Documentation/devicetree/bindings/interrupt-controller/apple,* F: Documentation/devicetree/bindings/iommu/apple,dart.yaml diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index f5b7bb4418d0dc..acd24294dedd33 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -94,6 +94,34 @@ }; }; +/* + * Mesa fingerprint sensor on SPI2. The power line (pin 108, active high) comes + * from the J313 platform device tree, which also lists an interrupt on pin 104. The + * interrupt is left out on purpose: the driver polls, and the trigger flags in + * the ADT (3) are not decoded with any certainty here. Pin 104 reads back as a + * plain input with a pull-down, which rules out the two active-low readings of + * that value but does not decide between the remaining ones. + * + * Do not add cs-gpios. The sensor needs the controller's native chip select, + * otherwise the SPI core skips the hardware CS timing. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 108 GPIO_ACTIVE_HIGH>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; + /* + * Per-device factory data written at manufacture. It is not + * redistributable and has to be extracted from the machine's own macOS + * install, so the name only fixes where the driver looks for it. + */ + firmware-name = "apple/mesacal-j313.bin"; +}; + &i2c1 { speaker_left: codec@31 { compatible = "ti,tas5770l", "ti,tas2770"; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 58c59075f2d2b7..517729436dccfa 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -861,6 +861,51 @@ status = "disabled"; }; + /* + * The J313 platform device tree puts the Mesa fingerprint sensor + * on SPI2. The 120 MHz parent was confirmed by timing transfers: + * a 22158-byte transfer requested at 8 MHz, divider 15, ran at + * 7.45 Mbit/s. A 200 MHz parent would make that divider + * 13.3 MHz, and the measurement could not then have come out + * below 8 MHz. + * + * The pin group was read out of the hardware rather than taken + * from a schematic. iBoot leaves every SPI pad it uses at + * peripheral function 1, and the three buses whose groups are + * already described here have the same register signature: two + * pads with no pull for CLK and MOSI, one with a pull-down for + * MISO. Pins 128, 129 and 130 are the only remaining pads with + * that signature, and they are contiguous, so they are SPI2. + * Pin 131 is a different pad type and is not part of the group. + * + * There is no chip-select pin. The platform device tree gives + * SPI1, SPI3 and SPI4 a GPIO chip select (pins 45, 49 and 24, each the + * last pad of its group) and gives SPI2 the string "null", so + * SPI2 uses the controller's own chip select. That is also why + * SPI0, which has no GPIO chip select either, lists three pins. + */ + spi2: spi@235108000 { + compatible = "apple,t8103-spi", "apple,spi"; + reg = <0x2 0x35108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_120m>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + mesa: fingerprint@0 { + compatible = "apple,mesa-fingerprint"; + reg = <0>; + spi-max-frequency = <8000000>; + spi-cpha; + status = "disabled"; + }; + }; + spi3: spi@23510c000 { compatible = "apple,t8103-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -1120,6 +1165,12 @@ ; }; + spi2_pins: spi2-pins { + pinmux = , /* CLK */ + , /* MOSI */ + ; /* MISO */ + }; + spi3_pins: spi3-pins { pinmux = , , diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 1215de82221d5d..ce17f810a635d2 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -888,6 +888,20 @@ status = "disabled"; }; + spi2: spi@235108000 { + compatible = "apple,t8112-spi", "apple,spi"; + reg = <0x2 0x35108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + spi3: spi@23510c000 { compatible = "apple,t8112-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -1291,6 +1305,13 @@ ; }; + spi2_pins: spi2-pins { + /* Three signal pads; SPI2 uses the controller's native chip select. */ + pinmux = , + , + ; + }; + spi3_pins: spi3-pins { pinmux = , , diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 856d3c5b1fa0d1..8b19cd52ff3a39 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -116,6 +116,7 @@ void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key, bio->bi_crypt_context = bc; } +EXPORT_SYMBOL_GPL(bio_crypt_set_ctx); void __bio_crypt_free_ctx(struct bio *bio) { @@ -186,6 +187,7 @@ bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, /* If the DUN wrapped through 0, don't treat it as contiguous. */ return carry == 0; } +EXPORT_SYMBOL_GPL(bio_crypt_dun_is_contiguous); /* * Checks that two bio crypt contexts are compatible - i.e. that @@ -349,6 +351,7 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key, return 0; } +EXPORT_SYMBOL_GPL(blk_crypto_init_key); bool blk_crypto_config_supported_natively(struct block_device *bdev, const struct blk_crypto_config *cfg) @@ -399,6 +402,7 @@ int blk_crypto_start_using_key(struct block_device *bdev, } return blk_crypto_fallback_start_using_mode(key->crypto_cfg.crypto_mode); } +EXPORT_SYMBOL_GPL(blk_crypto_start_using_key); /** * blk_crypto_evict_key() - Evict a blk_crypto_key from a block_device diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index 31974c7dd20c91..de0fa14096286c 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -127,6 +127,7 @@ config NVME_APPLE depends on OF && BLOCK depends on APPLE_RTKIT && APPLE_SART depends on ARCH_APPLE || COMPILE_TEST + select BLK_INLINE_ENCRYPTION select NVME_CORE help This provides support for the NVMe controller embedded in Apple SoCs diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index a3343c49c6af60..21f8f02f4cc356 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -10,6 +10,8 @@ */ #include +#include +#include #include #include #include @@ -208,6 +210,7 @@ struct apple_nvme { unsigned long flush_interval; unsigned long last_flush; struct delayed_work flush_dwork; + struct blk_crypto_profile crypto_profile; }; unsigned int flush_interval = 1000; @@ -217,6 +220,9 @@ MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); +#define APPLE_NVME_CRYPTO_KEY_SIZE 64 +#define APPLE_NVME_CRYPTO_DATA_UNIT_SIZE 4096 + static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl) { return container_of(ctrl, struct apple_nvme, ctrl); @@ -321,9 +327,12 @@ static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, - struct nvme_command *cmd) + struct nvme_command *cmd, + struct request *req) { struct apple_nvme *anv = queue_to_apple_nvme(q); + const u8 *key = NULL; + u64 dun = 0; u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; @@ -340,6 +349,22 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; + if (unlikely(req->crypt_ctx)) { + const struct blk_crypto_key *blk_key = req->crypt_ctx->bc_key; + + key = blk_key->bytes; + dun = req->crypt_ctx->bc_dun[0]; + } + + if (key) { + __le64 dun_le = cpu_to_le64(dun); + + tcb->dma_flags |= BIT(2); + memcpy(tcb->aes_iv, &dun_le, sizeof(dun_le)); + memset(tcb->_aes_unk, 0, sizeof(tcb->_aes_unk)); + memcpy(tcb->_aes_unk, key, 4); + memcpy(tcb->_aes_unk + 16, key + 16, 48); + } memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); /* @@ -592,9 +617,18 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) { struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); struct apple_nvme *anv = queue_to_apple_nvme(iod->q); + struct apple_nvmmu_tcb *tcb; + u32 tag; if (blk_rq_nr_phys_segments(req)) apple_nvme_unmap_data(anv, req); + tag = nvme_tag_from_cid(iod->cmd.common.command_id); + tcb = &iod->q->tcbs[tag]; + if (unlikely(tcb->dma_flags & BIT(2))) { + memzero_explicit(tcb->aes_iv, sizeof(tcb->aes_iv)); + memzero_explicit(tcb->_aes_unk, sizeof(tcb->_aes_unk)); + tcb->dma_flags &= ~BIT(2); + } } static void apple_nvme_complete_rq(struct request *req) @@ -820,6 +854,20 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, ret = nvme_setup_cmd(ns, req); if (ret) return ret; + if (unlikely(req->crypt_ctx)) { + const struct blk_crypto_key *key = req->crypt_ctx->bc_key; + + if (WARN_ON_ONCE(key->size != APPLE_NVME_CRYPTO_KEY_SIZE || + key->crypto_cfg.crypto_mode != + BLK_ENCRYPTION_MODE_AES_256_XTS || + key->crypto_cfg.key_type != + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED || + key->crypto_cfg.data_unit_size != + APPLE_NVME_CRYPTO_DATA_UNIT_SIZE)) { + ret = BLK_STS_NOTSUPP; + goto out_free_cmd; + } + } if (blk_rq_nr_phys_segments(req)) { ret = apple_nvme_map_data(anv, req, cmnd); @@ -835,7 +883,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, } if (anv->hw->has_lsq_nvmmu) - apple_nvme_submit_cmd_t8103(q, cmnd); + apple_nvme_submit_cmd_t8103(q, cmnd, req); else apple_nvme_submit_cmd_t8015(q, cmnd); @@ -1627,6 +1675,18 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto put_dev; } + if (anv->hw->has_lsq_nvmmu) { + ret = devm_blk_crypto_profile_init(dev, &anv->crypto_profile, 0); + if (ret) + goto put_dev; + anv->crypto_profile.max_dun_bytes_supported = sizeof(u64); + anv->crypto_profile.key_types_supported = + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED; + anv->crypto_profile.modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] = + BIT(ilog2(APPLE_NVME_CRYPTO_DATA_UNIT_SIZE)); + anv->crypto_profile.dev = dev; + } + ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | NVME_QUIRK_ADMIN_PAGE_ALIGN); @@ -1635,6 +1695,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto put_dev; } + if (anv->hw->has_lsq_nvmmu) + anv->ctrl.crypto_profile = &anv->crypto_profile; + return anv; put_dev: apple_nvme_detach_genpd(anv); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index d6a8aac8e72f1d..883a284a8522c6 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4174,6 +4174,11 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) ns->ctrl = ctrl; kref_init(&ns->kref); +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + if (ctrl->crypto_profile) + blk_crypto_register(ctrl->crypto_profile, ns->queue); +#endif + if (nvme_init_ns_head(ns, info)) goto out_cleanup_disk; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 4732c4c5c149e9..5deb9d1067393a 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -354,6 +354,9 @@ struct nvme_ctrl { int numa_node; struct blk_mq_tag_set *tagset; struct blk_mq_tag_set *admin_tagset; +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct blk_crypto_profile *crypto_profile; +#endif struct list_head namespaces; struct mutex namespaces_lock; struct srcu_struct srcu; diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 41bd8fbc87aab3..c6062a8e0751fc 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -83,6 +83,7 @@ config RUST_APPLE_RTKIT depends on PM depends on RUST select APPLE_RTKIT + select RUST_APPLE_MAILBOX config APPLE_AOP tristate "Apple \"Always-on\" Processor" @@ -96,18 +97,34 @@ config APPLE_AOP Say 'y' here if you have an Apple laptop. +config APPLE_SEP_FV_INTERFACE + bool + config APPLE_SEP - tristate "Apple Secure Element Processor" - depends on ARCH_APPLE || COMPILE_TEST - depends on PM + tristate "Apple SEP (Secure Enclave Processor)" + depends on ARCH_APPLE depends on RUST - select RUST_APPLE_RTKIT - select RUST_APPLE_MAILBOX + depends on HW_RANDOM + depends on TRUSTED_KEYS + depends on CRYPTO + select CRYPTO_HMAC + select CRYPTO_SHA256 + select CRYPTO_AES + select CRYPTO_GCM + select CRYPTO_ECDH + select APPLE_SEP_FV_INTERFACE help - A security co-processor persent on Apple SoCs, controlling transparent - disk encryption, secure boot, HDCP, biometric auth and probably more. + Driver for the Apple SEP (Secure Enclave Processor) on Apple silicon. + It drives the Touch ID fingerprint sensor over a /dev/sep-bio character + device (enrol and match), exposes the SEP hardware RNG, and registers a + SEP-backed trusted key source so keyctl can seal keys to the enclave. - Say 'y' here if you have an Apple SoC. + The driver reads the existing machine-wide xART gigalocker directly from + the iBoot system container, locating it by its root records. It validates + the complete store and both root records before registering with SEP. + Writes remain disabled unless the xart_writes module parameter is set. + + Say Y here if you have an Apple silicon Mac. config APPLE_PMP tristate "Apple Power Management Processor" diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 1baa6de9449589..76aa178f98f755 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -22,6 +22,10 @@ apple-tunable-y = tunable.o obj-$(CONFIG_APPLE_AOP) += aop.o -obj-$(CONFIG_APPLE_SEP) += sep.o +obj-$(CONFIG_APPLE_SEP_FV_INTERFACE) += fv_shim.o + +obj-$(CONFIG_APPLE_SEP) += apple-sep.o +apple-sep-y := sep.o hwrng_shim.o store_shim.o bio_shim.o sha_shim.o \ + crypto_shim.o sensor_shim.o p256_shim.o trusted_shim.o work_shim.o obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs new file mode 100644 index 00000000000000..e66710b6935cf3 --- /dev/null +++ b/drivers/soc/apple/bio.rs @@ -0,0 +1,911 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use crate::store::{Key, Store}; +use kernel::ioctl::{_IO, _IOR, _IOW, _IOWR}; +use kernel::prelude::*; +use kernel::uaccess::{UserPtr, UserSlice}; + +pub(crate) const IFACE_VERSION: u32 = 4; + +pub(crate) const ENROL_USER_ID: i32 = 1000; + +pub(crate) const UUID_LEN: usize = 16; +const LABEL_LEN: usize = 128; +const NONCE_LEN: usize = 32; +pub(crate) const TOKEN_LEN: usize = 32; +const MAX_IDENTITIES: usize = 32; + +pub(crate) const DEVICE_NAME: &CStr = c"sep-bio"; +pub(crate) const DEVICE_MODE: u16 = 0o600; + +#[derive(Clone, Copy, Default)] +#[repr(u32)] +enum State { + #[default] + Idle = 0, + Pending = 1, + Progress = 2, + Done = 3, + Failed = 4, +} + +#[derive(Clone, Copy, Default, PartialEq)] +#[repr(u32)] +pub(crate) enum Guidance { + #[default] + None = 0, + Place = 1, + LiftAndMove = 2, + HoldStill = 3, +} + +#[derive(Clone, Copy, Default)] +#[repr(u32)] +enum MatchResult { + #[default] + NoMatch = 0, + Match = 1, + NotCompared = 2, +} + +pub(crate) const ENROL_STAGES: u32 = 8; + +const TOKEN_LIFETIME_NS: u64 = 10 * 1_000_000_000; + +const SUSPEND_SLACK_NS: u64 = 1_000_000; + +const STATUS_LOCAL: u32 = 0; + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct Identity { + uuid: [u8; UUID_LEN], + label: [u8; LABEL_LEN], +} + +impl Identity { + const ZERO: Identity = Identity { + uuid: [0; UUID_LEN], + label: [0; LABEL_LEN], + }; +} + +#[repr(C)] +#[derive(Default)] +struct Info { + version: u32, + sensor_present: u32, + enrolled: u32, + capacity: u32, + enroll_stages: u32, + reserved: [u32; 3], +} + +#[repr(C)] +struct List { + count: u32, + reserved: u32, + id: [Identity; MAX_IDENTITIES], +} + +#[repr(C)] +struct EnrolStart { + flags: u32, + reserved: u32, + label: [u8; LABEL_LEN], +} + +#[repr(C)] +#[derive(Default)] +struct EnrolPoll { + state: State, + stage: u32, + stages_total: u32, + status: u32, + uuid: [u8; UUID_LEN], + guidance: Guidance, + progress_percent: u32, +} + +#[repr(C)] +struct VerifyStart { + flags: u32, + reserved: u32, + nonce: [u8; NONCE_LEN], +} + +#[repr(C)] +#[derive(Default)] +struct VerifyPoll { + state: State, + result: MatchResult, + status: u32, + reserved: u32, + uuid: [u8; UUID_LEN], + token: [u8; TOKEN_LEN], + deadline_ns: u64, +} + +#[repr(C)] +struct Delete { + uuid: [u8; UUID_LEN], +} + +pub(crate) const ATTEST_CHALLENGE_LEN: usize = 32; +/// Uncompressed P-256 public point (`04‖X‖Y`). +pub(crate) const ATTEST_PUB_LEN: usize = 65; +/// Max DER `SEQUENCE { INTEGER r, INTEGER s }` for P-256. +pub(crate) const ATTEST_SIG_MAX: usize = 72; + +#[repr(C)] +pub(crate) struct Attest { + pub(crate) sig_len: u32, + /// in: challenge to sign. + pub(crate) challenge: [u8; ATTEST_CHALLENGE_LEN], + /// out: ref-key public point. + pub(crate) public: [u8; ATTEST_PUB_LEN], + /// out: DER signature, zero-padded to `ATTEST_SIG_MAX`. + pub(crate) signature: [u8; ATTEST_SIG_MAX], + pub(crate) reserved: [u8; 3], +} + +// Sizes are baked into the ioctl numbers. +static_assert!(core::mem::size_of::() == 144); +static_assert!(core::mem::size_of::() == 32); +static_assert!(core::mem::size_of::() == 4616); +static_assert!(core::mem::size_of::() == 136); +static_assert!(core::mem::size_of::() == 40); +static_assert!(core::mem::size_of::() == 40); +static_assert!(core::mem::size_of::() == 72); +static_assert!(core::mem::size_of::() == 16); +static_assert!(core::mem::size_of::() == 176); + +macro_rules! ioctl_pod { + (to_user: $($t:ty),*; from_user: $($u:ty),*) => { + $( + // SAFETY: no padding, so the value is a faithful byte image. + unsafe impl kernel::transmute::AsBytes for $t {} + )* + $( + // SAFETY: every bit pattern of the members is a valid value. + unsafe impl kernel::transmute::FromBytes for $u {} + )* + }; +} +ioctl_pod! { + to_user: Info, List, EnrolPoll, VerifyPoll, Attest; + from_user: EnrolStart, VerifyStart, Delete, Attest +} + +const MAGIC: u32 = 0xB1; + +const IOC_GET_INFO: u32 = _IOR::(MAGIC, 0x01); +const IOC_LIST: u32 = _IOR::(MAGIC, 0x02); +const IOC_ENROL_START: u32 = _IOW::(MAGIC, 0x03); +const IOC_ENROL_POLL: u32 = _IOR::(MAGIC, 0x04); +const IOC_VERIFY_START: u32 = _IOW::(MAGIC, 0x05); +const IOC_VERIFY_POLL: u32 = _IOR::(MAGIC, 0x06); +const IOC_CANCEL: u32 = _IO(MAGIC, 0x07); +const IOC_DELETE: u32 = _IOW::(MAGIC, 0x08); +const IOC_DELETE_ALL: u32 = _IO(MAGIC, 0x09); +pub(crate) const IOC_ATTEST: u32 = _IOWR::(MAGIC, 0x0a); + +pub(crate) struct MatchEvidence { + identity: [u8; UUID_LEN], +} + +impl MatchEvidence { + pub(crate) fn from_enclave_reply(identity: [u8; UUID_LEN]) -> MatchEvidence { + MatchEvidence { identity } + } +} + +pub(crate) enum VerifyOutcome { + Failed(u32), + NoMatch, + Matched(MatchEvidence), +} + +struct ResultToken { + bytes: [u8; TOKEN_LEN], + nonce: [u8; NONCE_LEN], + identity: [u8; UUID_LEN], + deadline_ns: u64, + minted_mono_ns: u64, + minted_boot_ns: u64, +} + +impl ResultToken { + fn mint( + evidence: &MatchEvidence, + nonce: &[u8; NONCE_LEN], + bytes: [u8; TOKEN_LEN], + ) -> ResultToken { + let mono = crate::shim::monotonic_ns(); + ResultToken { + bytes, + nonce: *nonce, + identity: evidence.identity, + deadline_ns: mono.saturating_add(TOKEN_LIFETIME_NS), + minted_mono_ns: mono, + minted_boot_ns: crate::shim::boottime_ns(), + } + } + + fn usable(&self, nonce: &[u8; NONCE_LEN], identity: &[u8; UUID_LEN]) -> bool { + let mono = crate::shim::monotonic_ns(); + if mono > self.deadline_ns { + return false; + } + let suspended = crate::shim::boottime_ns().saturating_sub(self.minted_boot_ns); + let awake = mono.saturating_sub(self.minted_mono_ns); + if suspended.saturating_sub(awake) > SUSPEND_SLACK_NS { + return false; + } + self.nonce == *nonce && self.identity == *identity + } +} + +const PRIVATE_TYPE_IDENTITIES: u8 = 0xF1; + +fn identity_key() -> Key { + Key::root(PRIVATE_TYPE_IDENTITIES) +} + +const INDEX_FORMAT: u32 = 2; + +pub(crate) struct IdentityIndex { + entries: KVec, +} + +impl IdentityIndex { + pub(crate) fn new() -> IdentityIndex { + IdentityIndex { + entries: KVec::new(), + } + } + + pub(crate) fn load(store: &mut Store) -> Result { + let mut index = IdentityIndex::new(); + let Some(raw) = store.read(&identity_key())? else { + return Ok(index); + }; + if raw.len() < 4 { + return Ok(index); + } + if u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]) != INDEX_FORMAT { + return Err(EINVAL); + } + if raw.len() < 8 { + return Ok(index); + } + let count = u32::from_le_bytes([raw[4], raw[5], raw[6], raw[7]]) as usize; + let stride = UUID_LEN + LABEL_LEN; + for i in 0..count.min(MAX_IDENTITIES) { + let at = 8 + i * stride; + if at + stride > raw.len() { + break; + } + let mut entry = Identity::ZERO; + entry.uuid.copy_from_slice(&raw[at..at + UUID_LEN]); + entry + .label + .copy_from_slice(&raw[at + UUID_LEN..at + stride]); + index.entries.push(entry, GFP_KERNEL)?; + } + Ok(index) + } + + fn save(&self, store: &mut Store) -> Result<()> { + let mut raw = KVec::new(); + raw.extend_from_slice(&INDEX_FORMAT.to_le_bytes(), GFP_KERNEL)?; + raw.extend_from_slice(&(self.entries.len() as u32).to_le_bytes(), GFP_KERNEL)?; + for entry in self.entries.iter() { + raw.extend_from_slice(&entry.uuid, GFP_KERNEL)?; + raw.extend_from_slice(&entry.label, GFP_KERNEL)?; + } + store.write(&identity_key(), &raw) + } + + pub(crate) fn total(&self) -> usize { + self.entries.len() + } + + pub(crate) fn insert(&mut self, id: Identity) -> Result<()> { + for existing in self.entries.iter_mut() { + if existing.uuid == id.uuid { + *existing = id; + return Ok(()); + } + } + if self.entries.len() >= MAX_IDENTITIES { + return Err(ENOSPC); + } + self.entries.push(id, GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn persist(&self, store: &mut Store) -> Result<()> { + self.save(store) + } + + pub(crate) fn identity_v1_for( + &self, + uuid: &[u8; UUID_LEN], + user_id: i32, + ) -> Option { + if !self.contains(uuid) { + return None; + } + Some(crate::sbio::IdentityV1::from_index(user_id, *uuid)) + } + + pub(crate) fn reconcile_to( + &mut self, + listed: &[[u8; UUID_LEN]], + ) -> Result> { + let mut dropped = KVec::new(); + for entry in self.entries.iter() { + if !listed.contains(&entry.uuid) { + dropped.push(entry.uuid, GFP_KERNEL)?; + } + } + self.entries.retain(|e| listed.contains(&e.uuid)); + for uuid in listed.iter() { + if !self.contains(uuid) { + self.insert(Identity { + uuid: *uuid, + label: [0u8; LABEL_LEN], + })?; + } + } + Ok(dropped) + } + + pub(crate) fn remove(&mut self, uuid: &[u8; UUID_LEN]) { + self.entries.retain(|e| e.uuid != *uuid); + } + + pub(crate) fn contains_uuid(&self, uuid: &[u8; UUID_LEN]) -> bool { + self.contains(uuid) + } + + fn contains(&self, uuid: &[u8; UUID_LEN]) -> bool { + self.entries.iter().any(|e| e.uuid == *uuid) + } +} + +enum Op { + Idle, + Enrol { + label: [u8; LABEL_LEN], + stage: u32, + terminal: Option, + guidance: Guidance, + percent: u32, + }, + Verify { + nonce: [u8; NONCE_LEN], + terminal: Option, + }, +} + +enum EnrolOutcome { + Failed(u32), + Done([u8; UUID_LEN]), +} + +impl Op { + fn is_terminal(&self) -> bool { + match self { + Op::Idle => false, + Op::Enrol { terminal, .. } => terminal.is_some(), + Op::Verify { terminal, .. } => terminal.is_some(), + } + } + + fn may_start(&self) -> bool { + matches!(self, Op::Idle) || self.is_terminal() + } +} + +pub(crate) struct Session { + open: bool, + op: Op, + unseen: bool, + token: Option, +} + +impl Session { + pub(crate) fn new() -> Session { + Session { + open: false, + op: Op::Idle, + unseen: false, + token: None, + } + } + + fn reset(&mut self) { + self.open = false; + self.op = Op::Idle; + self.unseen = false; + self.token = None; + } + + pub(crate) fn ready(&self) -> bool { + self.unseen + } +} + +pub(crate) struct Context<'a> { + pub(crate) session: &'a mut Session, + pub(crate) index: &'a IdentityIndex, + pub(crate) sensor_present: bool, +} + +pub(crate) struct Handled { + pub(crate) ret: isize, + pub(crate) wake: bool, + pub(crate) start_enrol: bool, + pub(crate) start_verify: bool, + pub(crate) delete_identity: Option, + pub(crate) delete_identities: KVec, +} + +fn ok() -> Result { + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +fn require_admin() -> Result<()> { + if crate::shim::capable_admin() { + Ok(()) + } else { + Err(EPERM) + } +} + +fn require_sensor(present: bool) -> Result<()> { + if present { + Ok(()) + } else { + Err(ENODEV) + } +} + +pub(crate) fn ioctl(ctx: &mut Context<'_>, cmd: u32, arg: usize) -> Result { + let user = UserPtr::from_addr(arg); + + match cmd { + IOC_GET_INFO => get_info(ctx, user), + IOC_LIST => list(ctx, user), + IOC_ENROL_START => enrol_start(ctx, user), + IOC_ENROL_POLL => enrol_poll(ctx, user), + IOC_VERIFY_START => verify_start(ctx, user), + IOC_VERIFY_POLL => verify_poll(ctx, user), + IOC_CANCEL => cancel(ctx), + IOC_DELETE => delete(ctx, user), + IOC_DELETE_ALL => delete_all(ctx), + _ => Err(ENOTTY), + } +} + +fn get_info(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let info = Info { + version: IFACE_VERSION, + sensor_present: u32::from(ctx.sensor_present), + enrolled: ctx.index.total() as u32, + capacity: MAX_IDENTITIES as u32, + enroll_stages: ENROL_STAGES, + reserved: [0; 3], + }; + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&info)?; + ok() +} + +fn list(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = List { + count: 0, + reserved: 0, + id: [Identity::ZERO; MAX_IDENTITIES], + }; + for entry in ctx.index.entries.iter() { + if out.count as usize >= MAX_IDENTITIES { + break; + } + out.id[out.count as usize] = *entry; + out.count += 1; + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + ok() +} + +fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { + require_admin()?; + let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + if request.flags != 0 || request.reserved != 0 { + return Err(EINVAL); + } + require_sensor(ctx.sensor_present)?; + + if !ctx.session.op.may_start() { + return Err(EBUSY); + } + if ctx.index.total() >= MAX_IDENTITIES { + return Err(ENOSPC); + } + + ctx.session.op = Op::Enrol { + label: request.label, + stage: 0, + terminal: None, + guidance: Guidance::Place, + percent: 0, + }; + Ok(Handled { + ret: 0, + wake: false, + start_enrol: true, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +pub(crate) fn enrol_advance( + session: &mut Session, + completed: u32, + percent_now: u32, + guide: Guidance, +) -> bool { + if let Op::Enrol { + stage, + terminal, + guidance, + percent, + .. + } = &mut session.op + { + if terminal.is_none() { + *stage = completed; + *percent = percent_now; + *guidance = guide; + session.unseen = true; + return true; + } + } + false +} + +pub(crate) fn enrol_guide(session: &mut Session, guide: Guidance) -> bool { + if let Op::Enrol { + terminal, guidance, .. + } = &mut session.op + { + if terminal.is_none() && *guidance != guide { + *guidance = guide; + session.unseen = true; + return true; + } + } + false +} + +pub(crate) fn enrol_finish( + session: &mut Session, + index: &mut IdentityIndex, + outcome: core::result::Result<[u8; UUID_LEN], u32>, +) -> bool { + let Op::Enrol { + label, terminal, .. + } = &mut session.op + else { + return false; + }; + if terminal.is_some() { + return false; + } + match outcome { + Ok(uuid) => { + match index.insert(Identity { + uuid, + label: *label, + }) { + Ok(()) => *terminal = Some(EnrolOutcome::Done(uuid)), + Err(_) => *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)), + } + } + Err(status) => *terminal = Some(EnrolOutcome::Failed(status)), + } + session.unseen = true; + true +} + +pub(crate) fn enrol_is_live(session: &Session) -> bool { + matches!(&session.op, Op::Enrol { terminal: None, .. }) && session.open +} + +fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = EnrolPoll { + stages_total: ENROL_STAGES, + ..EnrolPoll::default() + }; + + match &ctx.session.op { + Op::Idle => out.state = State::Idle, + Op::Verify { .. } => return Err(EBUSY), + Op::Enrol { + stage, + terminal, + guidance, + percent, + .. + } => { + out.stage = *stage; + out.guidance = *guidance; + out.progress_percent = *percent; + match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending + } + } + Some(EnrolOutcome::Failed(status)) => { + out.state = State::Failed; + out.status = *status; + } + Some(EnrolOutcome::Done(uuid)) => { + out.state = State::Done; + out.uuid = *uuid; + } + } + } + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + ctx.session.unseen = false; + ok() +} + +fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let request: VerifyStart = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + if request.flags != 0 || request.reserved != 0 { + return Err(EINVAL); + } + require_sensor(ctx.sensor_present)?; + + if !ctx.session.op.may_start() { + return Err(EBUSY); + } + + if ctx.index.total() == 0 { + ctx.session.token = None; + ctx.session.op = Op::Verify { + nonce: request.nonce, + terminal: Some(VerifyOutcome::NoMatch), + }; + ctx.session.unseen = true; + return Ok(Handled { + ret: 0, + wake: true, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }); + } + + ctx.session.token = None; + ctx.session.op = Op::Verify { + nonce: request.nonce, + terminal: None, + }; + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: true, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +pub(crate) fn verify_finish( + session: &mut Session, + outcome: VerifyOutcome, + token_bytes: [u8; TOKEN_LEN], +) -> bool { + let Op::Verify { nonce, terminal } = &mut session.op else { + return false; + }; + if terminal.is_some() { + return false; + } + + let minted = match &outcome { + VerifyOutcome::Matched(evidence) => Some(ResultToken::mint(evidence, nonce, token_bytes)), + VerifyOutcome::NoMatch | VerifyOutcome::Failed(_) => None, + }; + + *terminal = Some(outcome); + session.token = minted; + session.unseen = true; + true +} + +pub(crate) fn verify_is_live(session: &Session) -> bool { + matches!(&session.op, Op::Verify { terminal: None, .. }) && session.open +} + +pub(crate) fn capture_is_live(session: &Session) -> bool { + enrol_is_live(session) || verify_is_live(session) +} + +fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { + let mut out = VerifyPoll::default(); + // Spend the token only after the copy to user succeeds; a fault must not lose a match. + let mut consume_token = false; + + match &ctx.session.op { + Op::Idle => out.state = State::Idle, + Op::Enrol { .. } => return Err(EBUSY), + Op::Verify { nonce, terminal } => match terminal { + None => { + out.state = if ctx.session.unseen { + State::Progress + } else { + State::Pending + } + } + Some(VerifyOutcome::Failed(status)) => { + out.state = State::Failed; + out.result = MatchResult::NotCompared; + out.status = *status; + } + Some(VerifyOutcome::NoMatch) => { + out.state = State::Done; + out.result = MatchResult::NoMatch; + } + Some(VerifyOutcome::Matched(evidence)) => { + out.state = State::Done; + match ctx.session.token.as_ref() { + Some(token) if token.usable(nonce, &evidence.identity) => { + out.result = MatchResult::Match; + out.uuid = evidence.identity; + out.token = token.bytes; + out.deadline_ns = token.deadline_ns; + consume_token = true; + } + _ => { + out.result = MatchResult::NotCompared; + } + } + } + }, + } + + UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + if consume_token { + ctx.session.token = None; + } + ctx.session.unseen = false; + ok() +} + +fn cancel(ctx: &mut Context<'_>) -> Result { + if ctx.session.op.is_terminal() { + return Err(ENOENT); + } + + match &mut ctx.session.op { + Op::Idle => return Err(ENOENT), + Op::Enrol { terminal, .. } => *terminal = Some(EnrolOutcome::Failed(STATUS_LOCAL)), + Op::Verify { terminal, .. } => *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)), + } + + ctx.session.token = None; + ctx.session.unseen = true; + + Ok(Handled { + ret: 0, + wake: true, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) +} + +fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { + require_admin()?; + let request: Delete = UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + + if !ctx.index.contains(&request.uuid) { + pr_info!( + "sep_bio: DELETE of an identity not held; reporting success (nothing to delete)\n" + ); + return ok(); + } + + // `0x57` takes one `identity_v1_t` (signed user id + 16-byte UUID). + let Some(identity) = ctx.index.identity_v1_for(&request.uuid, ENROL_USER_ID) else { + pr_warn!("sep_bio: DELETE of a held identity could not be expressed as an identity_v1_t\n"); + return Err(ENOENT); + }; + + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: Some(identity), + delete_identities: KVec::new(), + }) +} + +fn delete_all(ctx: &mut Context<'_>) -> Result { + require_admin()?; + + if ctx.index.total() == 0 { + pr_info!("sep_bio: DELETE_ALL on empty index; reporting success (nothing to delete)\n"); + return ok(); + } + + let mut ids = KVec::new(); + for entry in ctx.index.entries.iter() { + let Some(identity) = ctx.index.identity_v1_for(&entry.uuid, ENROL_USER_ID) else { + pr_warn!( + "sep_bio: DELETE_ALL of a held identity could not be expressed as an identity_v1_t\n" + ); + return Err(ENOENT); + }; + ids.push(identity, GFP_KERNEL)?; + } + + Ok(Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: ids, + }) +} + +pub(crate) fn open(session: &mut Session) -> Result<()> { + if session.open { + return Err(EBUSY); + } + session.reset(); + session.open = true; + Ok(()) +} + +pub(crate) fn release(session: &mut Session) { + session.reset(); +} diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c new file mode 100644 index 00000000000000..50f7e7ddf61ea7 --- /dev/null +++ b/drivers/soc/apple/bio_shim.c @@ -0,0 +1,203 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +struct sep_bio_chardev { + struct miscdevice misc; + wait_queue_head_t wq; + struct mutex lock; + void *ctx; /* NULL once detached */ + int (*f_open)(void *ctx); + void (*f_release)(void *ctx); + long (*f_ioctl)(void *ctx, unsigned int cmd, unsigned long arg); + int (*f_ready)(void *ctx); + bool registered; + int open_count; +}; + +static struct sep_bio_chardev *from_file(struct file *file) +{ + return container_of(file->private_data, struct sep_bio_chardev, misc); +} + +static int sep_bio_fop_open(struct inode *inode, struct file *file) +{ + struct sep_bio_chardev *d = from_file(file); + int ret; + + mutex_lock(&d->lock); + if (!d->ctx) { + mutex_unlock(&d->lock); + return -ENODEV; + } + ret = d->f_open(d->ctx); + if (!ret) + d->open_count++; + mutex_unlock(&d->lock); + + return ret; +} + +static int sep_bio_fop_release(struct inode *inode, struct file *file) +{ + struct sep_bio_chardev *d = from_file(file); + bool last; + + mutex_lock(&d->lock); + if (d->ctx) + d->f_release(d->ctx); + d->open_count--; + last = (d->open_count == 0) && !d->ctx; + mutex_unlock(&d->lock); + + /* Detached while this file was open: nobody else can reach it now. */ + if (last) { + mutex_destroy(&d->lock); + kfree(d); + } + + return 0; +} + +static long sep_bio_fop_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct sep_bio_chardev *d = from_file(file); + long ret; + + mutex_lock(&d->lock); + ret = d->ctx ? d->f_ioctl(d->ctx, cmd, arg) : -ENODEV; + mutex_unlock(&d->lock); + + return ret; +} + +/* + * Readable when a *_POLL ioctl has something the caller has not seen yet; quiet + * once that ioctl consumes it. + */ +static __poll_t sep_bio_fop_poll(struct file *file, + struct poll_table_struct *wait) +{ + struct sep_bio_chardev *d = from_file(file); + + __poll_t mask; + + poll_wait(file, &d->wq, wait); + + mutex_lock(&d->lock); + if (!d->ctx) + mask = EPOLLERR | EPOLLHUP; + else + mask = d->f_ready(d->ctx) ? (EPOLLIN | EPOLLRDNORM) : 0; + mutex_unlock(&d->lock); + + return mask; +} + +static const struct file_operations sep_bio_fops = { + .owner = THIS_MODULE, + .open = sep_bio_fop_open, + .release = sep_bio_fop_release, + .unlocked_ioctl = sep_bio_fop_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .poll = sep_bio_fop_poll, +}; + +/* @name must outlive the registration; the Rust side passes a &'static CStr. */ +void *sep_bio_register(const char *name, unsigned short mode, void *ctx, + int (*f_open)(void *), + void (*f_release)(void *), + long (*f_ioctl)(void *, unsigned int, unsigned long), + int (*f_ready)(void *)) +{ + struct sep_bio_chardev *d; + int ret; + + d = kzalloc(sizeof(*d), GFP_KERNEL); + if (!d) + return NULL; + + init_waitqueue_head(&d->wq); + mutex_init(&d->lock); + d->ctx = ctx; + d->f_open = f_open; + d->f_release = f_release; + d->f_ioctl = f_ioctl; + d->f_ready = f_ready; + + d->misc.minor = MISC_DYNAMIC_MINOR; + d->misc.name = name; + d->misc.fops = &sep_bio_fops; + d->misc.mode = mode; + + ret = misc_register(&d->misc); + if (ret) { + mutex_destroy(&d->lock); + kfree(d); + return NULL; + } + d->registered = true; + + return d; +} + +void sep_bio_unregister(void *dev) +{ + struct sep_bio_chardev *d = dev; + bool free_now; + + if (!d) + return; + + if (d->registered) + misc_deregister(&d->misc); + + mutex_lock(&d->lock); + d->ctx = NULL; + free_now = (d->open_count == 0); + mutex_unlock(&d->lock); + + wake_up_interruptible(&d->wq); + + if (free_now) { + mutex_destroy(&d->lock); + kfree(d); + } +} + +void sep_bio_wake(void *dev) +{ + struct sep_bio_chardev *d = dev; + + if (d) + wake_up_interruptible(&d->wq); +} + +/* In C so the CAP_SYS_ADMIN number stays a kernel header constant, not a Rust literal. */ +int sep_bio_capable_admin(void) +{ + return capable(CAP_SYS_ADMIN) ? 1 : 0; +} + +__u64 sep_bio_monotonic_ns(void) +{ + return ktime_get_ns(); +} + +__u64 sep_bio_boottime_ns(void) +{ + return ktime_get_boottime_ns(); +} diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs new file mode 100644 index 00000000000000..8b18efb4ead770 --- /dev/null +++ b/drivers/soc/apple/catacomb.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! File-backed persistence for the SEP catacombs. +//! +//! A catacomb is the opaque identity blob the enclave returns from `0x6c`; each +//! lives in its own variable-length file, too large for the block store. + +use crate::shim; +use crate::store::crc16_ccitt_false; +use kernel::prelude::*; + +const MAGIC: [u8; 8] = *b"SEPCTMB1"; +/// magic(8) + len(4, LE) + crc(2, LE) + pad(2) +const HEADER: usize = 16; +const MAX_CATACOMB: usize = 1 << 20; + +fn path_for(kind: u8) -> Option<&'static CStr> { + Some(match kind { + crate::PRIVATE_TYPE_CATACOMB_MASTER => c"/var/lib/apple-sep-catacomb-master.bin", + crate::PRIVATE_TYPE_CATACOMB_OWNER => c"/var/lib/apple-sep-catacomb-owner.bin", + crate::PRIVATE_TYPE_CATACOMB_USER => c"/var/lib/apple-sep-catacomb-user.bin", + _ => return None, + }) +} + +pub(crate) fn is_kind(kind: u8) -> bool { + path_for(kind).is_some() +} + +pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { + let path = path_for(kind).ok_or(EINVAL)?; + if blob.len() > MAX_CATACOMB { + return Err(ENOSPC); + } + let mut head = [0u8; HEADER]; + head[..MAGIC.len()].copy_from_slice(&MAGIC); + head[8..12].copy_from_slice(&(blob.len() as u32).to_le_bytes()); + head[12..14].copy_from_slice(&crc16_ccitt_false(blob).to_le_bytes()); + + let file = shim::StoreFile::open_trunc(path)?; + file.write_all(0, &head)?; + if !blob.is_empty() { + file.write_all(HEADER as u64, blob)?; + } + file.sync() +} + +pub(crate) fn read(kind: u8) -> Option> { + let path = path_for(kind)?; + let file = shim::StoreFile::open_readonly(path).ok()?; + let size = file.size().ok()?; + if size < HEADER as u64 { + return None; + } + let mut head = [0u8; HEADER]; + file.read_exact(0, &mut head).ok()?; + if head[..MAGIC.len()] != MAGIC { + return None; + } + let len = u32::from_le_bytes([head[8], head[9], head[10], head[11]]) as usize; + let crc = u16::from_le_bytes([head[12], head[13]]); + if len == 0 || len > MAX_CATACOMB || size != (HEADER + len) as u64 { + return None; + } + let mut blob: KVec = KVec::with_capacity(len, GFP_KERNEL).ok()?; + blob.resize(len, 0, GFP_KERNEL).ok()?; + file.read_exact(HEADER as u64, &mut blob).ok()?; + if crc16_ccitt_false(&blob) != crc { + return None; + } + Some(blob) +} diff --git a/drivers/soc/apple/control.rs b/drivers/soc/apple/control.rs new file mode 100644 index 00000000000000..beb6b608415de4 --- /dev/null +++ b/drivers/soc/apple/control.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use crate::proto; +use kernel::prelude::*; + +const MAX_INFLIGHT: usize = 8; + +const POOL_LEN: usize = (proto::TAG_POOL_LAST - proto::TAG_POOL_FIRST + 1) as usize; + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + tag: u8, + reply: Option, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + tag: 0, + reply: None, + }; +} + +struct EntropySink { + value: Option, + busy: bool, +} + +pub(crate) enum Delivery { + Entropy, + Matched, + Unmatched, +} + +pub(crate) struct ControlState { + slots: [Slot; MAX_INFLIGHT], + next_tag: u8, + retired: [u64; 4], + retired_count: u32, + entropy: EntropySink, +} + +impl ControlState { + pub(crate) fn new() -> Self { + ControlState { + slots: [Slot::FREE; MAX_INFLIGHT], + next_tag: proto::TAG_POOL_FIRST, + retired: [0; 4], + retired_count: 0, + entropy: EntropySink { + value: None, + busy: false, + }, + } + } + + fn is_retired(&self, tag: u8) -> bool { + self.retired[(tag >> 6) as usize] & (1u64 << (tag & 0x3f)) != 0 + } + + fn retire(&mut self, tag: u8) { + if !self.is_retired(tag) { + self.retired[(tag >> 6) as usize] |= 1u64 << (tag & 0x3f); + self.retired_count += 1; + } + } + + fn reclaim_retired(&mut self, tag: u8) -> bool { + if !self.is_retired(tag) { + return false; + } + self.retired[(tag >> 6) as usize] &= !(1u64 << (tag & 0x3f)); + self.retired_count -= 1; + true + } + + fn tag_in_flight(&self, tag: u8) -> bool { + self.slots.iter().any(|s| s.used && s.tag == tag) + } + + pub(crate) fn alloc(&mut self) -> Result<(usize, u8)> { + let idx = self.slots.iter().position(|s| !s.used).ok_or(EBUSY)?; + + for _ in 0..POOL_LEN { + let tag = self.next_tag; + self.next_tag = if tag >= proto::TAG_POOL_LAST { + proto::TAG_POOL_FIRST + } else { + tag + 1 + }; + if !self.tag_in_flight(tag) && !self.is_retired(tag) { + self.slots[idx] = Slot { + used: true, + tag, + reply: None, + }; + return Ok((idx, tag)); + } + } + Err(EBUSY) + } + + pub(crate) fn take_reply(&mut self, idx: usize) -> Option { + self.slots[idx].reply.take() + } + + pub(crate) fn release(&mut self, idx: usize) { + self.slots[idx] = Slot::FREE; + } + + // Retire the tag so a late reply cannot be mistaken for a later request's answer. + pub(crate) fn abandon(&mut self, idx: usize) -> u8 { + let tag = self.slots[idx].tag; + self.retire(tag); + self.slots[idx] = Slot::FREE; + tag + } + + pub(crate) fn retired_count(&self) -> u32 { + self.retired_count + } + + pub(crate) fn deliver(&mut self, reply: proto::ControlReply) -> Delivery { + // The reserved entropy tag is claimed first, outstanding request or not. + if reply.tag == proto::TAG_ENTROPY { + self.entropy.value = Some(reply.data_lo); + return Delivery::Entropy; + } + + if let Some(slot) = self + .slots + .iter_mut() + .find(|s| s.used && s.tag == reply.tag && s.reply.is_none()) + { + slot.reply = Some(reply); + return Delivery::Matched; + } + + if self.reclaim_retired(reply.tag) { + return Delivery::Unmatched; + } + + // No match: drop it, never hand it to another waiter. + Delivery::Unmatched + } + + pub(crate) fn entropy_begin(&mut self) -> Result<()> { + if self.entropy.busy { + return Err(EBUSY); + } + self.entropy.busy = true; + self.entropy.value = None; + Ok(()) + } + + pub(crate) fn entropy_take(&mut self) -> Option { + self.entropy.value.take() + } + + pub(crate) fn entropy_end(&mut self) { + self.entropy.busy = false; + } +} diff --git a/drivers/soc/apple/crypto_shim.c b/drivers/soc/apple/crypto_shim.c new file mode 100644 index 00000000000000..be7336ba005399 --- /dev/null +++ b/drivers/soc/apple/crypto_shim.c @@ -0,0 +1,287 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +#define SEP_GCM_TAG_LEN 16 + +int sep_random_bytes(void *buf, size_t len) +{ + int ret = wait_for_random_bytes(); + + if (ret) + return ret; + get_random_bytes(buf, len); + return 0; +} + +/* + * AES-GCM with a 16-byte (non-96-bit) IV, over the raw AES block cipher. The + * kernel's gcm(aes) only takes a 12-byte IV; the SEP's ECIES uses a 16-byte IV, + * whose J0 must be derived as GHASH_H(IV ‖ len-block), not IV‖0x00000001. + */ + +/* GF(2^128) multiply, GCM bit convention (big-endian block, R = 0xe1‖0^120). */ +static void sep_gf_mult(const u8 *X, const u8 *Y, u8 *out) +{ + u8 Z[16] = {0}; + u8 V[16]; + int i, j; + + memcpy(V, Y, 16); + for (i = 0; i < 128; i++) { + if ((X[i >> 3] >> (7 - (i & 7))) & 1) + for (j = 0; j < 16; j++) + Z[j] ^= V[j]; + { + int lsb = V[15] & 1; + + for (j = 15; j > 0; j--) + V[j] = (V[j] >> 1) | ((V[j - 1] & 1) << 7); + V[0] >>= 1; + if (lsb) + V[0] ^= 0xe1; + } + } + memcpy(out, Z, 16); +} + +/* GHASH: fold `len` bytes of `data` (zero-padded to 16) into accumulator Y. */ +static void sep_ghash(const u8 H[16], u8 Y[16], const u8 *data, size_t len) +{ + u8 blk[16], t[16]; + size_t i, n, j; + + for (i = 0; i < len; i += 16) { + n = (len - i >= 16) ? 16 : (len - i); + memset(blk, 0, 16); + memcpy(blk, data + i, n); + for (j = 0; j < 16; j++) + Y[j] ^= blk[j]; + sep_gf_mult(Y, H, t); + memcpy(Y, t, 16); + } +} + +/* Increment the low 32 bits (big-endian) of a counter block, in place. */ +static void sep_inc32(u8 cb[16]) +{ + u32 c = ((u32)cb[12] << 24) | ((u32)cb[13] << 16) | + ((u32)cb[14] << 8) | (u32)cb[15]; + c++; + cb[12] = c >> 24; + cb[13] = c >> 16; + cb[14] = c >> 8; + cb[15] = c; +} + +/* + * AES-GCM in place over `buf` = [aadlen AAD][datalen payload][16 tag], 16-byte + * IV, keylen 16 or 32. Returns 0, or -EBADMSG on a decrypt tag mismatch. + */ +static int sep_gcm16(int encrypt, const void *key, size_t keylen, + const u8 *iv, size_t aadlen, u8 *buf, size_t datalen) +{ + struct aes_enckey aes; + u8 H[16], J0[16], EJ0[16], S[16], cb[16], ks[16], lb[16], zero[16]; + u8 *ct; + u64 aadbits, ctbits; + size_t i, n, j; + int rc; + + rc = aes_prepareenckey(&aes, key, keylen); + if (rc) + return rc; + + /* H = AES_K(0^128) */ + memset(zero, 0, 16); + aes_encrypt(&aes, H, zero); + + /* J0 = GHASH_H(IV ‖ [0^64 ‖ len(IV)_64]) for a non-96-bit IV. */ + memset(J0, 0, 16); + sep_ghash(H, J0, iv, 16); + memset(lb, 0, 16); + lb[15] = 0x80; /* 128 bits, big-endian in the low 64 bits */ + for (j = 0; j < 16; j++) + J0[j] ^= lb[j]; + { + u8 t[16]; + + sep_gf_mult(J0, H, t); + memcpy(J0, t, 16); + } + aes_encrypt(&aes, EJ0, J0); + + ct = buf + aadlen; + + /* GHASH over AAD, then ciphertext (before CTR on decrypt). */ + memset(S, 0, 16); + sep_ghash(H, S, buf, aadlen); + if (!encrypt) + sep_ghash(H, S, ct, datalen); + + /* CTR keystream from inc32(J0). */ + memcpy(cb, J0, 16); + sep_inc32(cb); + for (i = 0; i < datalen; i += 16) { + aes_encrypt(&aes, ks, cb); + n = (datalen - i >= 16) ? 16 : (datalen - i); + for (j = 0; j < n; j++) + ct[i + j] ^= ks[j]; + sep_inc32(cb); + } + + if (encrypt) + sep_ghash(H, S, ct, datalen); + + /* lengths block: (aadbits)_64 ‖ (ctbits)_64, big-endian. */ + aadbits = (u64)aadlen * 8; + ctbits = (u64)datalen * 8; + memset(lb, 0, 16); + for (j = 0; j < 8; j++) + lb[7 - j] = (u8)(aadbits >> (8 * j)); + for (j = 0; j < 8; j++) + lb[15 - j] = (u8)(ctbits >> (8 * j)); + for (j = 0; j < 16; j++) + S[j] ^= lb[j]; + { + u8 t[16]; + + sep_gf_mult(S, H, t); + memcpy(S, t, 16); + } + for (j = 0; j < 16; j++) + S[j] ^= EJ0[j]; /* S is now the computed tag */ + + if (encrypt) { + memcpy(ct + datalen, S, 16); + rc = 0; + } else { + u8 diff = 0; + + for (j = 0; j < 16; j++) + diff |= S[j] ^ ct[datalen + j]; + rc = diff ? -EBADMSG : 0; + } + + memzero_explicit(ks, sizeof(ks)); + memzero_explicit(EJ0, sizeof(EJ0)); + memzero_explicit(H, sizeof(H)); + memzero_explicit(&aes, sizeof(aes)); + return rc; +} + +/* HMAC-SHA256 over one message; writes 32 bytes to `out`, untouched on error. */ +int sep_hmac_sha256(const void *key, size_t keylen, + const void *data, size_t datalen, u8 *out) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + int rc; + + tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + rc = crypto_shash_setkey(tfm, key, keylen); + if (rc) + goto out_tfm; + + desc = kzalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + rc = -ENOMEM; + goto out_tfm; + } + desc->tfm = tfm; + + rc = crypto_shash_digest(desc, data, datalen, out); + + kfree_sensitive(desc); +out_tfm: + crypto_free_shash(tfm); + return rc; +} + +/* + * AES-256-GCM in place over `buf` = [aadlen AAD][payload][tag]. Encrypt appends + * the tag; decrypt expects it and leaves plaintext in the first `datalen` bytes + * after the AAD. keylen 16 or 32, ivlen 12 or 16 (a 16-byte IV is routed to the + * GHASH J0 derivation, not truncated). + */ +int sep_gcm(int encrypt, const void *key, size_t keylen, + const void *iv, size_t ivlen, size_t aadlen, + void *buf, size_t buflen, size_t datalen) +{ + struct crypto_aead *tfm; + struct aead_request *req; + struct scatterlist sg; + DECLARE_CRYPTO_WAIT(wait); + size_t cryptlen; + u8 ivcopy[16]; + int rc; + + if ((keylen != 16 && keylen != 32) || + (ivlen != 12 && ivlen != 16) || ivlen > sizeof(ivcopy)) + return -EINVAL; + + if (aadlen + datalen + SEP_GCM_TAG_LEN < aadlen || + aadlen + datalen + SEP_GCM_TAG_LEN > buflen) + return -EINVAL; + + /* A 16-byte IV needs the GHASH J0 derivation; gcm(aes) would silently + * use only its first 12 bytes. */ + if (ivlen == 16) + return sep_gcm16(encrypt, key, keylen, iv, aadlen, buf, + datalen); + + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + rc = crypto_aead_setkey(tfm, key, keylen); + if (rc) + goto out_tfm; + rc = crypto_aead_setauthsize(tfm, SEP_GCM_TAG_LEN); + if (rc) + goto out_tfm; + + req = aead_request_alloc(tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto out_tfm; + } + + /* The API may modify the IV buffer, so it never sees the caller's. */ + memcpy(ivcopy, iv, ivlen); + + sg_init_one(&sg, buf, aadlen + datalen + SEP_GCM_TAG_LEN); + + /* AEAD API quirk: cryptlen includes the tag on decrypt but not on encrypt. */ + cryptlen = encrypt ? datalen : datalen + SEP_GCM_TAG_LEN; + + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + aead_request_set_ad(req, aadlen); + aead_request_set_crypt(req, &sg, &sg, cryptlen, ivcopy); + + rc = crypto_wait_req(encrypt ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); + + aead_request_free(req); + memzero_explicit(ivcopy, sizeof(ivcopy)); +out_tfm: + crypto_free_aead(tfm); + return rc; +} diff --git a/drivers/soc/apple/der.rs b/drivers/soc/apple/der.rs new file mode 100644 index 00000000000000..fcd198561872f4 --- /dev/null +++ b/drivers/soc/apple/der.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The key-store configuration blob, which is DER. + +use kernel::prelude::*; + +const TAG_SET: u8 = 0x31; +const TAG_SEQUENCE: u8 = 0x30; +const TAG_UTF8: u8 = 0x0c; +pub(crate) const TAG_INTEGER: u8 = 0x02; +pub(crate) const TAG_OCTET_STRING: u8 = 0x04; + +const MAX_ITEM: usize = 4096; + +fn put_header(out: &mut KVec, tag: u8, len: usize) -> Result<()> { + out.push(tag, GFP_KERNEL)?; + if len < 0x80 { + out.push(len as u8, GFP_KERNEL)?; + } else if len <= 0xff { + out.push(0x81, GFP_KERNEL)?; + out.push(len as u8, GFP_KERNEL)?; + } else if len <= 0xffff { + out.push(0x82, GFP_KERNEL)?; + out.push((len >> 8) as u8, GFP_KERNEL)?; + out.push((len & 0xff) as u8, GFP_KERNEL)?; + } else { + return Err(EINVAL); + } + Ok(()) +} + +fn integer_bytes(v: u32) -> Result> { + let mut raw = KVec::new(); + let be = v.to_be_bytes(); + let mut i = 0usize; + while i < 3 && be[i] == 0 { + i += 1; + } + if be[i] & 0x80 != 0 { + raw.push(0, GFP_KERNEL)?; + } + raw.extend_from_slice(&be[i..], GFP_KERNEL)?; + Ok(raw) +} + +#[derive(Clone, Copy)] +pub(crate) enum RefKeyValue<'a> { + /// op mnemonic: `oc`/`ow`/`ouw`. + Utf8(&'a [u8]), + Integer(u32), + Octets(&'a [u8]), + Der(&'a [u8]), +} + +/// Canonical DER `SET OF SEQUENCE`, members sorted by key — the shape the +/// enclave parser requires. +pub(crate) fn encode_refkey_set(items: &[(&[u8], RefKeyValue<'_>)]) -> Result> { + let mut members: KVec<(&[u8], KVec)> = KVec::new(); + for (key, value) in items { + if key.is_empty() || key.len() > MAX_ITEM { + return Err(EINVAL); + } + let mut inner = KVec::new(); + put_header(&mut inner, TAG_UTF8, key.len())?; + inner.extend_from_slice(key, GFP_KERNEL)?; + match value { + RefKeyValue::Utf8(s) => { + if s.len() > MAX_ITEM { + return Err(EINVAL); + } + put_header(&mut inner, TAG_UTF8, s.len())?; + inner.extend_from_slice(s, GFP_KERNEL)?; + } + RefKeyValue::Integer(v) => { + let raw = integer_bytes(*v)?; + put_header(&mut inner, TAG_INTEGER, raw.len())?; + inner.extend_from_slice(&raw, GFP_KERNEL)?; + } + RefKeyValue::Octets(b) => { + if b.len() > MAX_ITEM { + return Err(EINVAL); + } + put_header(&mut inner, TAG_OCTET_STRING, b.len())?; + inner.extend_from_slice(b, GFP_KERNEL)?; + } + RefKeyValue::Der(d) => inner.extend_from_slice(d, GFP_KERNEL)?, + } + let mut seq = KVec::new(); + put_header(&mut seq, TAG_SEQUENCE, inner.len())?; + seq.extend_from_slice(&inner, GFP_KERNEL)?; + members.push((key, seq), GFP_KERNEL)?; + } + members.sort_unstable_by(|a, b| a.0.cmp(b.0)); + let mut body = KVec::new(); + for (_, seq) in members.iter() { + body.extend_from_slice(seq, GFP_KERNEL)?; + } + let mut out = KVec::new(); + put_header(&mut out, TAG_SET, body.len())?; + out.extend_from_slice(&body, GFP_KERNEL)?; + Ok(out) +} + +fn take_tlv(buf: &[u8]) -> Option<(u8, &[u8], &[u8])> { + let tag = *buf.first()?; + let first = *buf.get(1)?; + let (len, off) = if first < 0x80 { + (first as usize, 2usize) + } else if first == 0x81 { + (*buf.get(2)? as usize, 3usize) + } else if first == 0x82 { + let hi = *buf.get(2)? as usize; + let lo = *buf.get(3)? as usize; + ((hi << 8) | lo, 4usize) + } else { + return None; + }; + if len > MAX_ITEM { + return None; + } + let end = off.checked_add(len)?; + if buf.len() < end { + return None; + } + Some((tag, &buf[off..end], &buf[end..])) +} + +pub(crate) fn refkey_find<'a>(set_blob: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + let mut body = match take_tlv(set_blob) { + Some((TAG_SET, body, [])) => body, + _ => return None, + }; + while !body.is_empty() { + let (seq_tag, seq, next) = take_tlv(body)?; + if seq_tag != TAG_SEQUENCE { + return None; + } + let (key_tag, this_key, after_key) = take_tlv(seq)?; + if key_tag != TAG_UTF8 { + return None; + } + if this_key == key { + let (_val_tag, _value, after_val) = take_tlv(after_key)?; + if !after_val.is_empty() { + return None; + } + return Some(&after_key[..after_key.len() - after_val.len()]); + } + body = next; + } + None +} + +pub(crate) fn octet_string_body(tlv: &[u8]) -> Option<&[u8]> { + match take_tlv(tlv) { + Some((TAG_OCTET_STRING, body, [])) => Some(body), + _ => None, + } +} diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs new file mode 100644 index 00000000000000..46dd70d15d03f5 --- /dev/null +++ b/drivers/soc/apple/dt.rs @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Device-tree work: bringing the SEP and its DART out of `status = "disabled"` +//! at module init, and the boot-persistent marker that stops the one-shot +//! registration from being sent twice. + +use kernel::bindings; +use kernel::error::to_result; +use kernel::prelude::*; + +const SEP_COMPATIBLE: &CStr = c"apple,sep"; +const IOMMUS_PROP: &CStr = c"iommus"; + +const STATUS_PROP: &CStr = c"status"; +const STATUS_OKAY: &CStr = c"okay"; + +// SEP DMA is offset BIT(40) above the DART input; one 64-bit value = two cells. +const DMA_OFFSET_PROP: &CStr = c"apple,dma-offset"; +// [0x100, 0] = BIT(40); [0x1, 0] would be BIT(32) and miss the page tables. +const DMA_OFFSET_CELLS: [u32; 2] = [0x100, 0x0]; + +const DMA_RANGE_PROP: &CStr = c"apple,dma-range"; +const DMA_RANGE_CELLS: [u32; 4] = [0, 0, 1, 0]; + +const REGISTERED_PROP: &CStr = c"apple,sep-shmem-registered-iova"; +const CHOSEN_PATH: &CStr = c"/chosen"; +const PREBOOT_UUID_PROP: &CStr = c"apfs-preboot-uuid"; + +pub(crate) struct DtNode(*mut bindings::device_node); + +impl DtNode { + fn find_compatible(compatible: &CStr) -> Option { + // SAFETY: `of_find_compatible_node` accepts NULL for `from` and `type`, + // takes a reference on the node it returns, and returns NULL when there + // is no match. + let np = unsafe { + bindings::of_find_compatible_node( + core::ptr::null_mut(), + core::ptr::null(), + compatible.as_char_ptr(), + ) + }; + (!np.is_null()).then_some(DtNode(np)) + } + + pub(crate) fn of_device(dev: &kernel::device::Device) -> Option { + // SAFETY: `dev` is valid for the duration of the borrow, and its + // `of_node` is either NULL or a live node the device holds a reference + // to. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return None; + } + // SAFETY: `np` is a live node; take our own reference for `DtNode`. + unsafe { bindings::of_node_get(np) }; + Some(DtNode(np)) + } + + fn parse_phandle(&self, name: &CStr, index: i32) -> Option { + // SAFETY: `self.0` is a valid node per the type invariant; + // `of_parse_phandle` takes a reference on what it returns and returns + // NULL when the property or the index is absent. + let np = unsafe { bindings::of_parse_phandle(self.0, name.as_char_ptr(), index) }; + (!np.is_null()).then_some(DtNode(np)) + } + + fn parent(&self) -> Option { + // SAFETY: `self.0` is a valid node per the type invariant; `of_get_parent` + // takes a reference on what it returns and returns NULL at the root. + let np = unsafe { bindings::of_get_parent(self.0) }; + (!np.is_null()).then_some(DtNode(np)) + } + + fn as_ptr(&self) -> *mut bindings::device_node { + self.0 + } + + fn is_available(&self) -> bool { + // SAFETY: `self.0` is a valid node per the type invariant. + unsafe { bindings::of_device_is_available(self.0) } + } + + pub(crate) fn has_property(&self, name: &CStr) -> bool { + // SAFETY: `self.0` is a valid node per the type invariant; a NULL + // length pointer is accepted. + let p = unsafe { + bindings::of_find_property(self.0, name.as_char_ptr(), core::ptr::null_mut()) + }; + !p.is_null() + } +} + +impl Drop for DtNode { + fn drop(&mut self) { + // SAFETY: we own one reference, taken by the function that produced + // `self.0`. + unsafe { bindings::of_node_put(self.0) }; + } +} + +fn with_changeset(f: F) -> Result<()> +where + F: FnOnce(*mut bindings::of_changeset) -> Result, +{ + let mut cs = core::mem::MaybeUninit::::uninit(); + let csp = cs.as_mut_ptr(); + + // SAFETY: `csp` points at valid uninitialised storage of the right type and + // `of_changeset_init` initialises it in place. + unsafe { bindings::of_changeset_init(csp) }; + + let res = match f(csp) { + // SAFETY: `csp` is an initialised changeset. + Ok(true) => to_result(unsafe { bindings::of_changeset_apply(csp) }), + Ok(false) => Ok(()), + Err(e) => Err(e), + }; + + // SAFETY: `csp` is an initialised changeset. + unsafe { bindings::of_changeset_destroy(csp) }; + + res +} + +fn queue_u32_array( + cs: *mut bindings::of_changeset, + node: &DtNode, + name: &CStr, + cells: &[u32], +) -> Result<()> { + // SAFETY: `cs` is an initialised changeset, `node` is a valid node, `name` + // is NUL-terminated, and `cells` is valid for `cells.len()` reads. The + // callee copies both the name and the value. + to_result(unsafe { + bindings::of_changeset_add_prop_u32_array( + cs, + node.as_ptr(), + name.as_char_ptr(), + cells.as_ptr(), + cells.len(), + ) + }) +} + +fn queue_status_okay(cs: *mut bindings::of_changeset, node: &DtNode) -> Result<()> { + // SAFETY: as above; `of_changeset_update_prop_string` duplicates the string. + to_result(unsafe { + bindings::of_changeset_update_prop_string( + cs, + node.as_ptr(), + STATUS_PROP.as_char_ptr(), + STATUS_OKAY.as_char_ptr(), + ) + }) +} + +pub(crate) fn sep_node() -> Option { + DtNode::find_compatible(SEP_COMPATIBLE) +} + +pub(crate) fn preboot_uuid() -> Option<[u8; 16]> { + // SAFETY: the path is NUL-terminated; a non-NULL result owns one node + // reference, which `DtNode` releases. + let raw = unsafe { + bindings::of_find_node_opts_by_path(CHOSEN_PATH.as_char_ptr(), core::ptr::null_mut()) + }; + let chosen = (!raw.is_null()).then_some(DtNode(raw))?; + let mut text = core::ptr::null(); + // SAFETY: `chosen` is live, the property name is NUL-terminated, and + // `text` is a valid output pointer. + if unsafe { + bindings::of_property_read_string( + chosen.as_ptr(), + PREBOOT_UUID_PROP.as_char_ptr(), + &mut text, + ) + } != 0 + { + return None; + } + + let mut uuid = bindings::uuid_t { b: [0; 16] }; + // SAFETY: the property is NUL-terminated and `uuid` is a valid output. + if unsafe { bindings::uuid_parse(text, &mut uuid) } != 0 || uuid.b.iter().all(|&b| b == 0) { + return None; + } + Some(uuid.b) +} + +pub(crate) fn enable_sep_and_dart() -> Result<()> { + let sep = sep_node().ok_or_else(|| { + pr_err!( + "apple_sep: no device-tree node with compatible '{}'\n", + SEP_COMPATIBLE + ); + ENODEV + })?; + + let dart = sep.parse_phandle(IOMMUS_PROP, 0).ok_or_else(|| { + pr_err!( + "apple_sep: SEP node has no '{}' phandle; cannot find its DART\n", + IOMMUS_PROP + ); + ENODEV + })?; + + let dart_available = dart.is_available(); + let need_offset = !dart.has_property(DMA_OFFSET_PROP); + let need_range = !dart.has_property(DMA_RANGE_PROP); + + pr_info!( + "apple_sep: SEP DART state: enabled={}, apple,dma-offset present={}, apple,dma-range present={}\n", + dart_available, + !need_offset, + !need_range + ); + + if dart_available && (need_offset || need_range) { + pr_err!( + "apple_sep: SEP DART already enabled but missing apple,dma-offset/apple,dma-range; the BIT(40) offset applies only at DART probe. Reboot and load this module first.\n" + ); + return Err(EBUSY); + } + + if need_offset || need_range || !dart_available { + with_changeset(|cs| { + let mut queued = false; + // Properties before status: the DART probes inside of_changeset_apply(). + if need_offset { + queue_u32_array(cs, &dart, DMA_OFFSET_PROP, &DMA_OFFSET_CELLS)?; + queued = true; + } + if need_range { + queue_u32_array(cs, &dart, DMA_RANGE_PROP, &DMA_RANGE_CELLS)?; + queued = true; + } + if !dart_available { + queue_status_okay(cs, &dart)?; + queued = true; + } + Ok(queued) + })?; + pr_info!( + "apple_sep: SEP DART changeset applied (offset BIT(40), range 0..4GiB, status okay)\n" + ); + } else { + pr_info!("apple_sep: SEP DART already enabled and configured\n"); + } + + if sep.is_available() { + pr_info!("apple_sep: SEP node already enabled\n"); + } else { + with_changeset(|cs| { + queue_status_okay(cs, &sep)?; + Ok(true) + })?; + pr_info!("apple_sep: SEP node enabled; platform device should now exist\n"); + } + + Ok(()) +} + +pub(crate) fn registration_already_sent(sep: &DtNode) -> bool { + sep.has_property(REGISTERED_PROP) +} + +pub(crate) fn mark_registration_sent(sep: &DtNode, iova: u64) -> Result<()> { + let cells = [(iova >> 32) as u32, iova as u32]; + with_changeset(|cs| { + queue_u32_array(cs, sep, REGISTERED_PROP, &cells)?; + Ok(true) + }) +} + +impl DtNode { + fn reg_base(&self) -> Option { + // SAFETY: `struct resource` is plain integers and pointers, so an + // all-zero value is valid. Zeroed rather than `default()` because + // bindgen's generated structs do not derive `Default`. + let mut res: bindings::resource = unsafe { core::mem::zeroed() }; + // SAFETY: `self.0` is a live node and `res` is a valid out-parameter. + let rc = unsafe { bindings::of_address_to_resource(self.0, 0, &mut res) }; + if rc != 0 { + None + } else { + Some(res.start) + } + } + + fn child_with_reg(&self, value: u32) -> Option { + let mut child: *mut bindings::device_node = core::ptr::null_mut(); + loop { + // SAFETY: `of_get_next_child` takes a live parent and the previous + // child, which it drops for us; NULL starts the iteration. + child = unsafe { bindings::of_get_next_child(self.0, child) }; + if child.is_null() { + return None; + } + let mut got: u32 = 0; + // SAFETY: `child` is live for this iteration and `got` is a valid + // out-parameter. + let rc = unsafe { + bindings::of_property_read_variable_u32_array( + child, + c"reg".as_char_ptr(), + &mut got, + 1, + 1, + ) + }; + if rc >= 0 && got == value { + return Some(DtNode(child)); + } + } + } +} + +fn node_at_address(base: u64) -> Option { + let mut np: *mut bindings::device_node = core::ptr::null_mut(); + loop { + // SAFETY: `of_find_node_with_property` accepts NULL to start and drops + // the reference to the previous node for us. + np = unsafe { bindings::of_find_node_with_property(np, c"reg".as_char_ptr()) }; + if np.is_null() { + return None; + } + let node = DtNode(np); + if node.reg_base() == Some(base) { + return Some(node); + } + } +} + +pub(crate) const SENSOR_COMPATIBLE: &CStr = c"apple,mesa-fingerprint"; + +const SENSOR_NODE_NAME: &CStr = c"mesa@0"; + +fn queue_sensor_node(cs_handle: *mut bindings::of_changeset, parent: &DtNode) -> Result<()> { + // SAFETY: `cs_handle` is a live changeset and `parent` a live node; the + // returned node belongs to the changeset, which owns it until applied. + let node = unsafe { + bindings::of_changeset_create_node(cs_handle, parent.0, SENSOR_NODE_NAME.as_char_ptr()) + }; + if node.is_null() { + pr_err!("apple_sep: could not create the sensor node\n"); + return Err(ENOMEM); + } + + // SAFETY: all four calls take the live changeset, the node just created and + // a NUL-terminated name; the u32 form takes a slice it copies. + let rc = unsafe { + let mut rc = bindings::of_changeset_add_prop_string( + cs_handle, + node, + c"compatible".as_char_ptr(), + SENSOR_COMPATIBLE.as_char_ptr(), + ); + // reg on an SPI child is the chip select. + if rc == 0 { + rc = add_u32(cs_handle, node, c"reg", 0); + } + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-max-frequency", 8_000_000); + } + // 20 ns setup/hold the controller does not apply itself. + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-cs-setup-delay-ns", 20); + } + if rc == 0 { + rc = add_u32(cs_handle, node, c"spi-cs-hold-delay-ns", 20); + } + // SPI mode 2 (CPOL=1, CPHA=0); absent spi-cpha is how CPHA=0 is spelled. + if rc == 0 { + rc = bindings::of_changeset_add_prop_bool(cs_handle, node, c"spi-cpol".as_char_ptr()); + } + rc + }; + if rc != 0 { + pr_err!("apple_sep: could not describe the sensor node: {}\n", rc); + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// # Safety +unsafe fn add_u32( + cs_handle: *mut bindings::of_changeset, + node: *mut bindings::device_node, + name: &CStr, + value: u32, +) -> i32 { + // of_changeset_add_prop_u32 is a static inline missing from the bindings; wrap the array form. + // SAFETY: per this function's contract; `value` is read as one element. + unsafe { + bindings::of_changeset_add_prop_u32_array(cs_handle, node, name.as_char_ptr(), &value, 1) + } +} + +pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { + // A machine that describes the sensor in its own device tree (compatible + // apple,mesa-fingerprint) is handled machine-agnostically: enable that node + // and its SPI controller wherever they sit, with the power/IRQ GPIOs coming + // from the node itself. Only when no such node exists do we fall back to + // creating one at the caller's fixed controller address. + if let Some(sensor) = DtNode::find_compatible(SENSOR_COMPATIBLE) { + let controller = sensor.parent().ok_or_else(|| { + pr_err!("apple_sep: sensor node has no parent SPI controller\n"); + ENODEV + })?; + pr_info!( + "apple_sep: sensor node present in the device tree; enabling it and its SPI bus\n" + ); + return with_changeset(|cs_handle| { + let mut queued = false; + if !controller.is_available() { + queue_status_okay(cs_handle, &controller)?; + queued = true; + } + if !sensor.is_available() { + queue_status_okay(cs_handle, &sensor)?; + queued = true; + } + Ok(queued) + }); + } + + let controller = node_at_address(base).ok_or_else(|| { + pr_err!( + "apple_sep: no device-tree node with reg base 0x{:x}; the sensor's SPI bus is not in this tree\n", + base + ); + ENODEV + })?; + + let controller_ok = controller.is_available(); + let existing = controller.child_with_reg(cs); + pr_info!( + "apple_sep: sensor SPI bus at 0x{:x}: controller enabled={}, chip-select {} child present={}\n", + base, + controller_ok, + cs, + existing.is_some() + ); + + if controller_ok && existing.is_some() { + return Ok(()); + } + + if existing.is_some() && !controller_ok { + // Enable only the bus; a second child would be two devices at one chip select. + pr_info!("apple_sep: sensor node already present; enabling only the bus\n"); + return with_changeset(|cs_handle| { + queue_status_okay(cs_handle, &controller)?; + Ok(true) + }); + } + + pr_info!( + "apple_sep: creating the sensor node (absent from Linux's tree; the firmware exposes it at /arm-io/spi2/mesa)\n" + ); + + with_changeset(|cs_handle| { + // Controller before child: the SPI core's notifier needs the bus to exist first. + if !controller_ok { + queue_status_okay(cs_handle, &controller)?; + } + queue_sensor_node(cs_handle, &controller)?; + Ok(true) + }) +} + +/// Resolve a reserved-memory region by name on the SEP node's device. +/// +/// The cold-boot path needs this: the firmware image is handed over in a +/// bootloader-injected `sepfw` region. +pub(crate) fn reserved_region(dev: &kernel::device::Device, name: &CStr) -> Result<(u64, usize)> { + // SAFETY: `dev` is valid and its `of_node` is either NULL or a live node + // the device holds a reference to. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return Err(ENODEV); + } + + // SAFETY: `struct resource` is plain integers with no pointer state, so an + // all-zero value is valid. + let mut res: bindings::resource = unsafe { core::mem::zeroed() }; + // SAFETY: `np` is a live node, `name` is NUL-terminated and `res` is a + // valid out-parameter. + let ret = unsafe { + bindings::of_reserved_mem_region_to_resource_byname(np, name.as_char_ptr(), &mut res) + }; + to_result(ret).map_err(|e| { + pr_err!( + "apple_sep: reserved-memory region '{}' is unavailable ({:?})\n", + name, + e + ); + e + })?; + + let size = res.end.wrapping_sub(res.start).wrapping_add(1); + if size == 0 || size > usize::MAX as u64 { + return Err(EINVAL); + } + Ok((res.start, size as usize)) +} diff --git a/drivers/soc/apple/fv.rs b/drivers/soc/apple/fv.rs new file mode 100644 index 00000000000000..9a9d3c759837da --- /dev/null +++ b/drivers/soc/apple/fv.rs @@ -0,0 +1,1142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use crate::{image, proto, SepData, SksRequest, PHASE_READY}; +use kernel::prelude::*; +use kernel::sync::atomic::Relaxed; + +const WRAPPED_KEY_LEN: usize = 40; +const OPAQUE_KEY_LEN: usize = 64; +const IV_KEY_LEN: usize = 16; +const SYSTEM_CLIENT: u64 = 1; +const NO_KEYBAG_HANDLE: i32 = -1; +const WRAPPED_KEY_FLAG: i32 = 1 << 1; +const VOLUME_KEY_FLAG: i32 = 1; +const SECRET_MAX_LEN: usize = 1024; +const RECORD_MAX_LEN: usize = 528; +const FILE_KEY_MAX_LEN: usize = 168; +const EPHEMERAL_KEY_MAX_LEN: usize = 64; +const FV_UNWRAP_VERSION: u32 = 0; +const FV_UNWRAP_OPTIONS: u32 = 2; +const PFK_UNWRAP_VERSION: u32 = 2; +const PFK_SYSTEM_VOLUME_HANDLE: i32 = -5; +const PFK_UNWRAP_OPTIONS: u32 = 0x102; +const PFK_KEY_FLAG: i32 = 1 << 1; +const PFK_NEW_VERSION: u32 = 2; +const PFK_NEW_OPTIONS: u32 = 0x102; +const PFK_NEW_KEY_FLAG: i32 = 1 << 1; +const PFK_FS_CONTEXT_LEN: usize = 28; +const FV_LOAD_CLASS_KEYS: u32 = 0x12; +const FV_UNLOAD_CLASS_KEYS: u32 = 0x13; +const FV_SYSTEM_VOLUME_OPTION: u64 = 4; +const FV_MAX_VOLUME_MAPS: usize = 32; +const FV_STATE_UUID_KEY: &[u8] = b"kid"; + +pub(crate) struct VolumeMap { + apfs_uuid: [u8; 16], + bag_uuid: [u8; 16], + refs: u32, +} + +#[repr(C)] +struct KernelKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv: [u8; IV_KEY_LEN], +} + +#[repr(C)] +struct KernelNewFileKey { + key: KernelKey, + wrapped_ekwk: [u8; FILE_KEY_MAX_LEN], + wrapped_ek: [u8; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: usize, + wrapped_ek_len: usize, +} + +#[repr(C)] +struct KernelOps { + unwrap_media_key: + unsafe extern "C" fn(*mut c_void, *const u8, usize, u32, *mut KernelKey) -> c_int, + unwrap_volume_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, + *mut KernelKey, + ) -> c_int, + load_class_keys: unsafe extern "C" fn( + *mut c_void, + *const u8, + *const u8, + usize, + *const u8, + usize, + *const u8, + usize, + ) -> c_int, + unload_class_keys: unsafe extern "C" fn(*mut c_void, *const u8, *const u8, usize) -> c_int, + unwrap_file_key: unsafe extern "C" fn( + *mut c_void, + *const u8, + u32, + *const u8, + usize, + *const u8, + usize, + *mut KernelKey, + ) -> c_int, + new_file_key: + unsafe extern "C" fn(*mut c_void, *const u8, u32, u64, u16, *mut KernelNewFileKey) -> c_int, +} + +extern "C" { + fn sep_fv_register_v2(context: *mut c_void, ops: *const KernelOps) -> c_int; + fn sep_fv_unregister_v2(context: *mut c_void); +} + +static KERNEL_OPS: KernelOps = KernelOps { + unwrap_media_key: kernel_unwrap_media_key, + unwrap_volume_key: kernel_unwrap_volume_key, + load_class_keys: kernel_load_class_keys, + unload_class_keys: kernel_unload_class_keys, + unwrap_file_key: kernel_unwrap_file_key, + new_file_key: kernel_new_file_key, +}; + +const PFK_VOLUME_PARAMS_PREFIX: [u8; 12] = [ + 0x31, 0x1a, 0x30, 0x18, 0x0c, 0x04, b'v', b'u', b'i', b'd', 0x04, 0x10, +]; +const PFK_VOLUME_PARAMS_LEN: usize = PFK_VOLUME_PARAMS_PREFIX.len() + 16; + +const FV_PARAMS_DER: [u8; 35] = [ + 0x31, 0x21, 0x30, 0x0a, 0x0c, 0x02, b'k', b'c', 0x04, 0x04, 0, 0, 0, 0, 0x30, 0x13, 0x0c, 0x07, + b'o', b'p', b't', b'i', b'o', b'n', b's', 0x04, 0x08, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +struct MediaKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv_key: [u8; IV_KEY_LEN], +} + +struct VolumeKey { + opaque: [u8; OPAQUE_KEY_LEN], +} + +struct FileKey { + opaque: [u8; OPAQUE_KEY_LEN], + iv_key: [u8; IV_KEY_LEN], +} + +struct NewFileKey { + key: FileKey, + wrapped_ekwk: [u8; FILE_KEY_MAX_LEN], + wrapped_ek: [u8; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: usize, + wrapped_ek_len: usize, +} + +impl Drop for NewFileKey { + fn drop(&mut self) { + image::wipe(&mut self.wrapped_ekwk); + image::wipe(&mut self.wrapped_ek); + } +} + +impl Drop for VolumeKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + } +} + +impl Drop for MediaKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + image::wipe(&mut self.iv_key); + } +} + +impl Drop for FileKey { + fn drop(&mut self) { + image::wipe(&mut self.opaque); + image::wipe(&mut self.iv_key); + } +} + +impl SepData { + fn pfk_params(volume_uuid: &[u8; 16]) -> [u8; PFK_VOLUME_PARAMS_LEN] { + let mut params = [0u8; PFK_VOLUME_PARAMS_LEN]; + params[..PFK_VOLUME_PARAMS_PREFIX.len()].copy_from_slice(&PFK_VOLUME_PARAMS_PREFIX); + params[PFK_VOLUME_PARAMS_PREFIX.len()..].copy_from_slice(volume_uuid); + params + } + + fn pfk_class(protection_class: u32) -> Result { + match protection_class & 0x1f { + 1..=4 => Ok(protection_class), + 6 => Ok((protection_class & !0x1f) | 13), + 7 => Ok((protection_class & !0x1f) | 17), + _ => Err(EINVAL), + } + } + + fn fv_options(&self, volume_uuid: &[u8; 16]) -> u64 { + if self.xarm.lock().os_uuid == Some(*volume_uuid) { + FV_SYSTEM_VOLUME_OPTION + } else { + 0 + } + } + + fn fv_params(options: u64) -> [u8; FV_PARAMS_DER.len()] { + let mut params = FV_PARAMS_DER; + let options_at = params.len() - 8; + params[options_at..].copy_from_slice(&options.to_le_bytes()); + params + } + + fn fv_ready(&self) -> Result<()> { + if self.phase.load(Relaxed) != PHASE_READY { + return Err(EAGAIN); + } + if !self.sks_ready() { + return Err(ENODEV); + } + Ok(()) + } + + fn resolve_fv_uuid(&self, apfs_uuid: &[u8; 16]) -> [u8; 16] { + self.fv_volumes + .lock() + .iter() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + .map_or(*apfs_uuid, |entry| entry.bag_uuid) + } + + fn retain_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result { + let mut volumes = self.fv_volumes.lock(); + if let Some(entry) = volumes + .iter_mut() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + { + if entry.bag_uuid != *bag_uuid { + return Err(EINVAL); + } + entry.refs = entry.refs.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(false); + } + if volumes.len() >= FV_MAX_VOLUME_MAPS { + return Err(ENOSPC); + } + volumes.push( + VolumeMap { + apfs_uuid: *apfs_uuid, + bag_uuid: *bag_uuid, + refs: 1, + }, + GFP_KERNEL, + )?; + Ok(true) + } + + fn fv_volume_is_last(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result { + let volumes = self.fv_volumes.lock(); + let entry = volumes + .iter() + .find(|entry| entry.apfs_uuid == *apfs_uuid) + .ok_or(ENOENT)?; + if entry.bag_uuid != *bag_uuid { + return Err(EINVAL); + } + Ok(entry.refs == 1) + } + + fn release_fv_volume(&self, apfs_uuid: &[u8; 16], bag_uuid: &[u8; 16]) -> Result<()> { + let mut volumes = self.fv_volumes.lock(); + let index = volumes + .iter() + .position(|entry| entry.apfs_uuid == *apfs_uuid) + .ok_or(ENOENT)?; + if volumes[index].bag_uuid != *bag_uuid { + return Err(EINVAL); + } + if volumes[index].refs > 1 { + volumes[index].refs -= 1; + } else { + volumes.swap_remove(index); + } + Ok(()) + } + + fn sks_req_fv_blob_state( + &self, + volume_uuid: &[u8; 16], + volume_key: &[u8], + ) -> Result { + let params = Self::fv_params(self.fv_options(volume_uuid)); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_GET_BLOB_STATE_NAME, + msg: crate::sks::encode_sks_get_blob_state(self.sks_next_seq(), len), + img, + }) + } + + fn fv_blob_uuid(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<[u8; 16]> { + let out = self + .sks_send(self.sks_req_fv_blob_state(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: GET_BLOB_STATE failed with status {}\n", + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_GET_BLOB_STATE_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + let version = fields.i32(); + if version != Some(0) { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE response has version {:?}, {} body bytes\n", + version, + body.len() + ); + return Err(EMSGSIZE); + } + let state = match fields.blob() { + Some(state) => state, + None => { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE has no complete state blob in {} body bytes\n", + body.len() + ); + return Err(EMSGSIZE); + } + }; + let uuid_tlv = match crate::der::refkey_find(state, FV_STATE_UUID_KEY) { + Some(uuid) => uuid, + None => { + let head0 = state + .get(..8) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0); + let head1 = state + .get(8..16) + .and_then(|bytes| bytes.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0); + dev_err!( + self.dev, + "fv: GET_BLOB_STATE returned {} state bytes without a UUID (head {:016x}{:016x})\n", + state.len(), + head0, + head1 + ); + return Err(EMSGSIZE); + } + }; + let raw_uuid = crate::der::octet_string_body(uuid_tlv).ok_or(EMSGSIZE)?; + if raw_uuid.len() != 16 { + dev_err!( + self.dev, + "fv: GET_BLOB_STATE returned a {}-byte UUID\n", + raw_uuid.len() + ); + return Err(EMSGSIZE); + } + let mut bag_uuid = [0u8; 16]; + bag_uuid.copy_from_slice(raw_uuid); + Ok(bag_uuid) + } + + pub(crate) fn register_fv_kernel(&self) -> Result<()> { + let context = core::ptr::from_ref(self).cast_mut().cast::(); + // SAFETY: `KERNEL_OPS` is static and `remove()` unregisters this + // pointer before the driver's `Arc` can be dropped. + kernel::error::to_result(unsafe { sep_fv_register_v2(context, &KERNEL_OPS) }) + } + + pub(crate) fn unregister_fv_kernel(&self) { + let context = core::ptr::from_ref(self).cast_mut().cast::(); + // SAFETY: the pointer is the one passed to `sep_fv_register_v2`. + unsafe { sep_fv_unregister_v2(context) }; + } + + fn sks_req_unwrap_media_key_from_class( + &self, + wrapped: &[u8; WRAPPED_KEY_LEN], + protection_class: u32, + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(1)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(wrapped)?; + body.put_i32(NO_KEYBAG_HANDLE)?; + body.put_u32(protection_class)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = + crate::sks::encode_sks_unwrap_media_key(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_MEDIA_KEY_NAME, + msg, + img, + }) + } + + fn decode_media_key(&self, out: crate::SksOutcome) -> Result { + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: UNWRAP_MEDIA_KEY failed with status {}\n", + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_MEDIA_KEY_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(1) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let iv_key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + + if key.len() != OPAQUE_KEY_LEN + || iv_key.len() != IV_KEY_LEN + || flags & WRAPPED_KEY_FLAG == 0 + { + return Err(EMSGSIZE); + } + + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + let mut iv = [0; IV_KEY_LEN]; + iv.copy_from_slice(iv_key); + Ok(MediaKey { opaque, iv_key: iv }) + } + + fn unwrap_media_key_from_class( + &self, + wrapped: &[u8; WRAPPED_KEY_LEN], + protection_class: u32, + ) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_media_key_from_class(wrapped, protection_class)) + .ok_or(EIO)?; + self.decode_media_key(out) + } + + fn sks_req_unwrap_vek(&self, secret: &[u8], kek: &[u8], vek: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(FV_UNWRAP_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(&FV_PARAMS_DER)?; + body.put_u32(FV_UNWRAP_OPTIONS)?; + body.put_blob(secret)?; + body.put_blob(kek)?; + body.put_blob(vek)?; + body.put_blob(&[])?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_unwrap_vek(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_VEK_NAME, + msg, + img, + }) + } + + fn unwrap_vek(&self, secret: &[u8], kek: &[u8], vek: &[u8]) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_vek(secret, kek, vek)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!(self.dev, "fv: UNWRAP_VEK failed with status {}\n", status); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_VEK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(FV_UNWRAP_VERSION as i32) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + if key.len() != OPAQUE_KEY_LEN || flags & VOLUME_KEY_FLAG == 0 { + return Err(EMSGSIZE); + } + + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + Ok(VolumeKey { opaque }) + } + + fn sks_req_unwrap_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + wrapped_ekwk: &[u8], + wrapped_ek: &[u8], + ) -> Result { + let class = Self::pfk_class(protection_class)?; + let bag_uuid = self.resolve_fv_uuid(volume_uuid); + let params = Self::pfk_params(&bag_uuid); + + let mut body = image::Body::new(); + body.put_u32(PFK_UNWRAP_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_i32(PFK_SYSTEM_VOLUME_HANDLE)?; + body.put_blob(¶ms)?; + body.put_u32(class)?; + body.put_blob(wrapped_ek)?; + body.put_blob(wrapped_ekwk)?; + body.put_blob(&[])?; + body.put_u32(PFK_UNWRAP_OPTIONS)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_UNWRAP_PFK_NAME, + msg: crate::sks::encode_sks_unwrap_pfk(self.sks_next_seq(), len), + img, + }) + } + + fn unwrap_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + wrapped_ekwk: &[u8], + wrapped_ek: &[u8], + ) -> Result { + self.fv_ready()?; + let out = self + .sks_send(self.sks_req_unwrap_file_key( + volume_uuid, + protection_class, + wrapped_ekwk, + wrapped_ek, + )) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: UNWRAP_PFK v2 class {} ekwk {} ek {} failed with status {}\n", + protection_class, + wrapped_ekwk.len(), + wrapped_ek.len(), + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_UNWRAP_PFK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(PFK_UNWRAP_VERSION as i32) { + return Err(EMSGSIZE); + } + let key = fields.blob().ok_or(EMSGSIZE)?; + let iv_key = fields.blob().ok_or(EMSGSIZE)?; + let _ephemeral_key = fields.blob().ok_or(EMSGSIZE)?; + let flags = fields.i32().ok_or(EMSGSIZE)?; + if key.len() != OPAQUE_KEY_LEN || iv_key.len() != IV_KEY_LEN || flags & PFK_KEY_FLAG == 0 { + return Err(EMSGSIZE); + } + let mut opaque = [0; OPAQUE_KEY_LEN]; + opaque.copy_from_slice(key); + let mut iv = [0; IV_KEY_LEN]; + iv.copy_from_slice(iv_key); + Ok(FileKey { opaque, iv_key: iv }) + } + + fn sks_req_new_file_key( + &self, + volume_uuid: &[u8; 16], + protection_class: u32, + ) -> Result { + let class = Self::pfk_class(protection_class)?; + let bag_uuid = self.resolve_fv_uuid(volume_uuid); + let params = Self::pfk_params(&bag_uuid); + let mut context = [0u8; PFK_FS_CONTEXT_LEN]; + context[10..26].copy_from_slice(&bag_uuid); + let mut body = image::Body::new(); + body.put_u32(PFK_NEW_VERSION)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_i32(PFK_SYSTEM_VOLUME_HANDLE)?; + body.put_blob(¶ms)?; + body.put_u32(class)?; + body.put_u32(PFK_NEW_OPTIONS)?; + body.put_blob(&context)?; + body.put_blob(&[])?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_NEW_PFK_NAME, + msg: crate::sks::encode_sks_new_pfk(self.sks_next_seq(), len), + img, + }) + } + + fn new_file_key(&self, volume_uuid: &[u8; 16], protection_class: u32) -> Result { + self.fv_ready()?; + let class = Self::pfk_class(protection_class)?; + let out = self + .sks_send(self.sks_req_new_file_key(volume_uuid, protection_class)) + .ok_or(EIO)?; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: NEW_PFK class {} failed with status {}\n", + protection_class, + status + ); + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_NEW_PFK_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(PFK_NEW_VERSION as i32) { + return Err(EMSGSIZE); + } + let opaque = fields.blob().ok_or(EMSGSIZE)?; + let iv = fields.blob().ok_or(EMSGSIZE)?; + let ephemeral = fields.blob().ok_or(EMSGSIZE)?; + let wrapped_ek = fields.blob().ok_or(EMSGSIZE)?; + let wrapped_ekwk = fields.blob().ok_or(EMSGSIZE)?; + let key_flags = fields.i32().ok_or(EMSGSIZE)?; + let wrapped_class = fields.i32().ok_or(EMSGSIZE)?; + if opaque.len() != OPAQUE_KEY_LEN + || iv.len() != IV_KEY_LEN + || ephemeral.is_empty() + || ephemeral.len() > EPHEMERAL_KEY_MAX_LEN + || wrapped_ek.is_empty() + || wrapped_ek.len() > FILE_KEY_MAX_LEN + || wrapped_ekwk.is_empty() + || wrapped_ekwk.len() > FILE_KEY_MAX_LEN + || key_flags & PFK_NEW_KEY_FLAG == 0 + || wrapped_class != class as i32 + { + dev_err!( + self.dev, + "fv: NEW_PFK malformed response: key {} iv {} eph {} ek {} ekwk {} flags {:#x} class {} expected {}\n", + opaque.len(), + iv.len(), + ephemeral.len(), + wrapped_ek.len(), + wrapped_ekwk.len(), + key_flags, + wrapped_class, + class + ); + return Err(EMSGSIZE); + } + + let mut key = FileKey { + opaque: [0; OPAQUE_KEY_LEN], + iv_key: [0; IV_KEY_LEN], + }; + key.opaque.copy_from_slice(opaque); + key.iv_key.copy_from_slice(iv); + let mut result = NewFileKey { + key, + wrapped_ekwk: [0; FILE_KEY_MAX_LEN], + wrapped_ek: [0; FILE_KEY_MAX_LEN], + wrapped_ekwk_len: wrapped_ekwk.len(), + wrapped_ek_len: wrapped_ek.len(), + }; + result.wrapped_ekwk[..wrapped_ekwk.len()].copy_from_slice(wrapped_ekwk); + result.wrapped_ek[..wrapped_ek.len()].copy_from_slice(wrapped_ek); + Ok(result) + } + + fn sks_req_load_class_keys( + &self, + volume_uuid: &[u8; 16], + secret: &[u8], + unlock_record: &[u8], + volume_key: &[u8], + ) -> Result { + let options = self.fv_options(volume_uuid); + let params = Self::fv_params(options); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_u32(FV_LOAD_CLASS_KEYS)?; + body.put_u64(options)?; + body.put_blob(secret)?; + body.put_blob(unlock_record)?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_SET_PROTECTION_NAME, + msg: crate::sks::encode_sks_set_protection(self.sks_next_seq(), len), + img, + }) + } + + fn load_class_keys( + &self, + volume_uuid: &[u8; 16], + secret: &[u8], + unlock_record: &[u8], + volume_key: &[u8], + ) -> Result<()> { + self.fv_ready()?; + let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; + let request = + self.sks_req_load_class_keys(volume_uuid, secret, unlock_record, volume_key)?; + if !self.retain_fv_volume(volume_uuid, &bag_uuid)? { + return Ok(()); + } + let out = match self.sks_send(Ok(request)) { + Some(out) => out, + None => { + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); + return Err(EIO); + } + }; + if out.reply.status != 0 { + let status: i32 = out.reply.status.into(); + dev_err!( + self.dev, + "fv: LOAD_CLASS_KEYS failed with status {}\n", + status + ); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); + return Err(EACCES); + } + let body = match self.sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) { + Some(body) => body, + None => { + self.rollback_class_keys(volume_uuid, volume_key); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); + return Err(EMSGSIZE); + } + }; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + self.rollback_class_keys(volume_uuid, volume_key); + let _ = self.release_fv_volume(volume_uuid, &bag_uuid); + return Err(EMSGSIZE); + } + let apfs_hi = u64::from_be_bytes(volume_uuid[..8].try_into().unwrap()); + let apfs_lo = u64::from_be_bytes(volume_uuid[8..].try_into().unwrap()); + let bag_hi = u64::from_be_bytes(bag_uuid[..8].try_into().unwrap()); + let bag_lo = u64::from_be_bytes(bag_uuid[8..].try_into().unwrap()); + dev_info!( + self.dev, + "fv: mapped APFS UUID {:016x}-{:016x} to keybag UUID {:016x}-{:016x}\n", + apfs_hi, + apfs_lo, + bag_hi, + bag_lo + ); + Ok(()) + } + + fn rollback_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) { + if let Ok(request) = self.sks_req_unload_class_keys(volume_uuid, volume_key) { + let _ = self.sks_send(Ok(request)); + } + } + + fn sks_req_unload_class_keys( + &self, + volume_uuid: &[u8; 16], + volume_key: &[u8], + ) -> Result { + let options = self.fv_options(volume_uuid); + let params = Self::fv_params(options); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(SYSTEM_CLIENT)?; + body.put_blob(¶ms)?; + body.put_u32(FV_UNLOAD_CLASS_KEYS)?; + body.put_u64(options)?; + body.put_blob(&[])?; + body.put_blob(&[])?; + body.put_blob(volume_key)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: crate::sks::SKS_SET_PROTECTION_NAME, + msg: crate::sks::encode_sks_set_protection(self.sks_next_seq(), len), + img, + }) + } + + fn unload_class_keys(&self, volume_uuid: &[u8; 16], volume_key: &[u8]) -> Result<()> { + self.fv_ready()?; + let bag_uuid = self.fv_blob_uuid(volume_uuid, volume_key)?; + if !self.fv_volume_is_last(volume_uuid, &bag_uuid)? { + return self.release_fv_volume(volume_uuid, &bag_uuid); + } + let result = (|| { + let out = self + .sks_send(self.sks_req_unload_class_keys(volume_uuid, volume_key)) + .ok_or(EIO)?; + if out.reply.status != 0 { + return Err(EACCES); + } + let body = self + .sks_report_response(crate::sks::SKS_SET_PROTECTION_NAME, &out) + .ok_or(EMSGSIZE)?; + let mut fields = proto::FieldCursor::new(body); + if fields.i32() != Some(0) || fields.blob().is_none() { + return Err(EMSGSIZE); + } + Ok(()) + })(); + let release = self.release_fv_volume(volume_uuid, &bag_uuid); + result.and(release) + } +} + +/// Borrow a C input buffer as a slice, rejecting null/empty/oversized inputs. +/// +/// # Safety +/// +/// When `ptr` is non-null and `len` is in `1..=max`, `ptr` must be valid for +/// reads of `len` bytes and that region must stay live and unmutated for the +/// whole of `'a`. The caller must pick `'a` so the returned slice cannot +/// outlive the underlying buffer. +unsafe fn input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { + if ptr.is_null() || len == 0 || len > max { + return Err(EINVAL); + } + // SAFETY: `ptr` is non-null and `len` is in `1..=max` here, so by this + // function's contract the region is valid for reads of `len` bytes and + // stays live for `'a`. + Ok(unsafe { core::slice::from_raw_parts(ptr, len) }) +} + +/// Borrow an optional C input buffer, treating a zero length as empty. +/// +/// # Safety +/// +/// Same obligations as [`input`]: when `len` is non-zero, `ptr` must be valid +/// for reads of `len` bytes and stay live for `'a`. +unsafe fn optional_input<'a>(ptr: *const u8, len: usize, max: usize) -> Result<&'a [u8]> { + if len == 0 { + return Ok(&[]); + } + // SAFETY: `len` is non-zero here; this function's contract is identical to + // `input`'s, so the caller already guarantees `ptr`/`len` are valid. + unsafe { input(ptr, len, max) } +} + +/// Borrow a C output slot for one key. +/// +/// # Safety +/// +/// When `ptr` is non-null it must be aligned, point to a live and writable +/// `KernelKey`, and grant exclusive access for the whole of `'a` (no other +/// alias may touch it while the returned reference lives). +unsafe fn output<'a>(ptr: *mut KernelKey) -> Result<&'a mut KernelKey> { + if ptr.is_null() { + return Err(EINVAL); + } + // SAFETY: `ptr` is non-null here, so by this function's contract it is an + // aligned, exclusively-owned, writable `KernelKey` that outlives `'a`. + Ok(unsafe { &mut *ptr }) +} + +/// Borrow a C output slot for one new-file-key result. +/// +/// # Safety +/// +/// When `ptr` is non-null it must be aligned, point to a live and writable +/// `KernelNewFileKey`, and grant exclusive access for the whole of `'a`. +unsafe fn new_file_output<'a>(ptr: *mut KernelNewFileKey) -> Result<&'a mut KernelNewFileKey> { + if ptr.is_null() { + return Err(EINVAL); + } + // SAFETY: `ptr` is non-null here, so by this function's contract it is an + // aligned, exclusively-owned, writable `KernelNewFileKey` that outlives + // `'a`. + Ok(unsafe { &mut *ptr }) +} + +/// Recover the driver state from a registered callback context pointer. +/// +/// # Safety +/// +/// When `context` is non-null it must be the pointer passed to +/// `sep_fv_register_v2` — a `*const SepData` taken from a live `Arc` — +/// and must still be registered, so the `SepData` is alive and only shared +/// (never uniquely) borrowed for the whole of `'a`. +unsafe fn sep<'a>(context: *mut c_void) -> Result<&'a SepData> { + if context.is_null() { + return Err(ENODEV); + } + // SAFETY: `context` is non-null here, so by this function's contract it + // points to a live `SepData` that stays valid and shared for `'a`. + Ok(unsafe { &*context.cast::() }) +} + +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `wrapped`/`wrapped_len` describe a buffer readable +/// for the duration of the call (or are null/0); and `key` points to aligned, +/// exclusively-owned, writable storage for one `KernelKey`. +unsafe extern "C" fn kernel_unwrap_media_key( + context: *mut c_void, + wrapped: *const u8, + wrapped_len: usize, + protection_class: u32, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. + let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `wrapped`/`wrapped_len` describe a + // buffer readable for the call; the borrow does not outlive it. + let wrapped = unsafe { input(wrapped, wrapped_len, WRAPPED_KEY_LEN)? }; + let wrapped: &[u8; WRAPPED_KEY_LEN] = wrapped.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. + let key = unsafe { output(key)? }; + let unwrapped = this.unwrap_media_key_from_class(wrapped, protection_class)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.copy_from_slice(&unwrapped.iv_key); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |()| 0) +} + +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; each `*const u8`/length pair describes a buffer +/// readable for the duration of the call (the optional `secret`/`unlock_record` +/// may be null with a zero length); and `key` points to aligned, +/// exclusively-owned, writable storage for one `KernelKey`. +unsafe extern "C" fn kernel_unwrap_volume_key( + context: *mut c_void, + secret: *const u8, + secret_len: usize, + unlock_record: *const u8, + unlock_record_len: usize, + volume_key: *const u8, + volume_key_len: usize, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. + let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `secret`/`secret_len` describe a + // buffer readable for the call, or are null/0. + let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + // SAFETY: per this function's contract `unlock_record`/`unlock_record_len` + // describe a buffer readable for the call, or are null/0. + let unlock_record = + unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. + let key = unsafe { output(key)? }; + let unwrapped = this.unwrap_vek(secret, unlock_record, volume_key)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.fill(0); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |()| 0) +} + +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and each +/// other `*const u8`/length pair describes a buffer readable for the duration +/// of the call (the optional `secret`/`unlock_record` may be null with a zero +/// length). +unsafe extern "C" fn kernel_load_class_keys( + context: *mut c_void, + volume_uuid: *const u8, + secret: *const u8, + secret_len: usize, + unlock_record: *const u8, + unlock_record_len: usize, + volume_key: *const u8, + volume_key_len: usize, +) -> c_int { + let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. + let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `secret`/`secret_len` describe a + // buffer readable for the call, or are null/0. + let secret = unsafe { optional_input(secret, secret_len, SECRET_MAX_LEN)? }; + // SAFETY: per this function's contract `unlock_record`/`unlock_record_len` + // describe a buffer readable for the call, or are null/0. + let unlock_record = + unsafe { optional_input(unlock_record, unlock_record_len, RECORD_MAX_LEN)? }; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + this.load_class_keys(volume_uuid, secret, unlock_record, volume_key) + })(); + result.map_or_else(|error| error.to_errno(), |()| 0) +} + +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and +/// `volume_key`/`volume_key_len` describe a buffer readable for the duration of +/// the call. +unsafe extern "C" fn kernel_unload_class_keys( + context: *mut c_void, + volume_uuid: *const u8, + volume_key: *const u8, + volume_key_len: usize, +) -> c_int { + let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. + let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `volume_key`/`volume_key_len` + // describe a buffer readable for the call. + let volume_key = unsafe { input(volume_key, volume_key_len, RECORD_MAX_LEN)? }; + this.unload_class_keys(volume_uuid, volume_key) + })(); + result.map_or_else(|error| error.to_errno(), |()| 0) +} + +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; each +/// `wrapped_*`/length pair describes a buffer readable for the duration of the +/// call; and `key` points to aligned, exclusively-owned, writable storage for +/// one `KernelKey`. +unsafe extern "C" fn kernel_unwrap_file_key( + context: *mut c_void, + volume_uuid: *const u8, + protection_class: u32, + wrapped_ekwk: *const u8, + wrapped_ekwk_len: usize, + wrapped_ek: *const u8, + wrapped_ek_len: usize, + key: *mut KernelKey, +) -> c_int { + let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. + let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `wrapped_ekwk`/`wrapped_ekwk_len` + // describe a buffer readable for the call. + let wrapped_ekwk = unsafe { input(wrapped_ekwk, wrapped_ekwk_len, FILE_KEY_MAX_LEN)? }; + // SAFETY: per this function's contract `wrapped_ek`/`wrapped_ek_len` + // describe a buffer readable for the call. + let wrapped_ek = unsafe { input(wrapped_ek, wrapped_ek_len, FILE_KEY_MAX_LEN)? }; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelKey`, valid for the duration of the call. + let key = unsafe { output(key)? }; + let unwrapped = + this.unwrap_file_key(volume_uuid, protection_class, wrapped_ekwk, wrapped_ek)?; + key.opaque.copy_from_slice(&unwrapped.opaque); + key.iv.copy_from_slice(&unwrapped.iv_key); + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |()| 0) +} + +/// # Safety +/// +/// Invoked only through the registered `sep_fv_ops` dispatch (which holds the +/// registration lock): `context` is the pointer passed to `sep_fv_register_v2` +/// and is still registered; `volume_uuid` points to 16 readable bytes; and +/// `key` points to aligned, exclusively-owned, writable storage for one +/// `KernelNewFileKey`. +unsafe extern "C" fn kernel_new_file_key( + context: *mut c_void, + volume_uuid: *const u8, + protection_class: u32, + crypto_id: u64, + key_revision: u16, + key: *mut KernelNewFileKey, +) -> c_int { + let result: Result<()> = (|| { + // SAFETY: per this function's contract `context` is the still-registered + // `SepData` pointer, alive for the duration of the call. + let this = unsafe { sep(context)? }; + // SAFETY: per this function's contract `volume_uuid` points to 16 + // readable bytes for the duration of the call. + let volume_uuid = unsafe { input(volume_uuid, 16, 16)? }; + let volume_uuid: &[u8; 16] = volume_uuid.try_into().map_err(|_| EINVAL)?; + // SAFETY: per this function's contract `key` is exclusive writable + // storage for one `KernelNewFileKey`, valid for the duration of the call. + let key = unsafe { new_file_output(key)? }; + if crypto_id == 0 || key_revision == 0 { + return Err(EINVAL); + } + let generated = this.new_file_key(volume_uuid, protection_class)?; + key.key.opaque.copy_from_slice(&generated.key.opaque); + key.key.iv.copy_from_slice(&generated.key.iv_key); + key.wrapped_ekwk.copy_from_slice(&generated.wrapped_ekwk); + key.wrapped_ek.copy_from_slice(&generated.wrapped_ek); + key.wrapped_ekwk_len = generated.wrapped_ekwk_len; + key.wrapped_ek_len = generated.wrapped_ek_len; + Ok(()) + })(); + result.map_or_else(|error| error.to_errno(), |()| 0) +} diff --git a/drivers/soc/apple/fv_shim.c b/drivers/soc/apple/fv_shim.c new file mode 100644 index 00000000000000..566852005c2f86 --- /dev/null +++ b/drivers/soc/apple/fv_shim.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +static DECLARE_RWSEM(apple_sep_fv_registration_lock); +static DEFINE_MUTEX(apple_sep_fv_call_lock); +static struct sep_fv_ops apple_sep_fv_ops; +static void *apple_sep_fv_context; + +int sep_fv_register_v2(void *context, const struct sep_fv_ops *ops) +{ + int ret = 0; + + if (!context || !ops || !ops->unwrap_media_key || + !ops->unwrap_volume_key || !ops->load_class_keys || + !ops->unload_class_keys || !ops->unwrap_file_key || + !ops->new_file_key) + return -EINVAL; + + down_write(&apple_sep_fv_registration_lock); + if (apple_sep_fv_context) + ret = -EBUSY; + else { + apple_sep_fv_ops = *ops; + apple_sep_fv_context = context; + } + up_write(&apple_sep_fv_registration_lock); + return ret; +} +EXPORT_SYMBOL_GPL(sep_fv_register_v2); + +void sep_fv_unregister_v2(void *context) +{ + down_write(&apple_sep_fv_registration_lock); + if (apple_sep_fv_context == context) { + apple_sep_fv_context = NULL; + memzero_explicit(&apple_sep_fv_ops, sizeof(apple_sep_fv_ops)); + } + up_write(&apple_sep_fv_registration_lock); +} +EXPORT_SYMBOL_GPL(sep_fv_unregister_v2); + +static int apple_sep_fv_call(int (*call)(const struct sep_fv_ops *, void *), + void *argument) +{ + int ret; + + down_read(&apple_sep_fv_registration_lock); + if (!apple_sep_fv_context) { + ret = -ENODEV; + goto out; + } + mutex_lock(&apple_sep_fv_call_lock); + ret = call(&apple_sep_fv_ops, argument); + mutex_unlock(&apple_sep_fv_call_lock); +out: + up_read(&apple_sep_fv_registration_lock); + return ret; +} + +struct media_key_call { + const u8 *wrapped; + size_t wrapped_len; + u32 protection_class; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_media_key(const struct sep_fv_ops *ops, void *argument) +{ + struct media_key_call *call = argument; + + return ops->unwrap_media_key(apple_sep_fv_context, call->wrapped, + call->wrapped_len, call->protection_class, + call->key); +} + +int apple_sep_fv_unwrap_media_key(const u8 *wrapped, size_t wrapped_len, + u32 protection_class, + struct apple_sep_fv_key *key) +{ + struct media_key_call call = { + .wrapped = wrapped, + .wrapped_len = wrapped_len, + .protection_class = protection_class, + .key = key, + }; + int ret; + + if (!wrapped || !wrapped_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_media_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_media_key); + +struct volume_key_call { + const u8 *volume_uuid; + const u8 *secret; + size_t secret_len; + const u8 *unlock_record; + size_t unlock_record_len; + const u8 *volume_key; + size_t volume_key_len; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_volume_key(const struct sep_fv_ops *ops, void *argument) +{ + struct volume_key_call *call = argument; + + return ops->unwrap_volume_key(apple_sep_fv_context, call->secret, + call->secret_len, call->unlock_record, + call->unlock_record_len, call->volume_key, + call->volume_key_len, call->key); +} + +int apple_sep_fv_unwrap_volume_key(const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key) +{ + struct volume_key_call call = { + .secret = secret, + .secret_len = secret_len, + .unlock_record = unlock_record, + .unlock_record_len = unlock_record_len, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + .key = key, + }; + int ret; + + if ((!secret && secret_len) || (secret && !secret_len) || + (!unlock_record && unlock_record_len) || + (unlock_record && !unlock_record_len) || + !volume_key || !volume_key_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_volume_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_volume_key); + +static int call_load_class_keys(const struct sep_fv_ops *ops, void *argument) +{ + struct volume_key_call *call = argument; + + return ops->load_class_keys(apple_sep_fv_context, call->volume_uuid, + call->secret, + call->secret_len, call->unlock_record, + call->unlock_record_len, call->volume_key, + call->volume_key_len); +} + +int apple_sep_fv_load_class_keys_v2(const u8 volume_uuid[16], + const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len) +{ + struct volume_key_call call = { + .volume_uuid = volume_uuid, + .secret = secret, + .secret_len = secret_len, + .unlock_record = unlock_record, + .unlock_record_len = unlock_record_len, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + }; + + if (!volume_uuid || (!secret && secret_len) || (secret && !secret_len) || + (!unlock_record && unlock_record_len) || + (unlock_record && !unlock_record_len) || + !volume_key || !volume_key_len) + return -EINVAL; + return apple_sep_fv_call(call_load_class_keys, &call); +} +EXPORT_SYMBOL_GPL(apple_sep_fv_load_class_keys_v2); + +struct unload_class_keys_call { + const u8 *volume_uuid; + const u8 *volume_key; + size_t volume_key_len; +}; + +static int call_unload_class_keys(const struct sep_fv_ops *ops, + void *argument) +{ + struct unload_class_keys_call *call = argument; + + return ops->unload_class_keys(apple_sep_fv_context, call->volume_uuid, + call->volume_key, + call->volume_key_len); +} + +int apple_sep_fv_unload_class_keys_v2(const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len) +{ + struct unload_class_keys_call call = { + .volume_uuid = volume_uuid, + .volume_key = volume_key, + .volume_key_len = volume_key_len, + }; + + if (!volume_uuid || !volume_key || !volume_key_len) + return -EINVAL; + return apple_sep_fv_call(call_unload_class_keys, &call); +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unload_class_keys_v2); + +struct file_key_call { + const u8 *volume_uuid; + u32 protection_class; + const u8 *wrapped_ekwk; + size_t wrapped_ekwk_len; + const u8 *wrapped_ek; + size_t wrapped_ek_len; + struct apple_sep_fv_key *key; +}; + +static int call_unwrap_file_key(const struct sep_fv_ops *ops, void *argument) +{ + struct file_key_call *call = argument; + + return ops->unwrap_file_key(apple_sep_fv_context, call->volume_uuid, + call->protection_class, call->wrapped_ekwk, + call->wrapped_ekwk_len, call->wrapped_ek, + call->wrapped_ek_len, call->key); +} + +int apple_sep_fv_unwrap_file_key(const u8 volume_uuid[16], + u32 protection_class, + const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, + const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key) +{ + struct file_key_call call = { + .volume_uuid = volume_uuid, + .protection_class = protection_class, + .wrapped_ekwk = wrapped_ekwk, + .wrapped_ekwk_len = wrapped_ekwk_len, + .wrapped_ek = wrapped_ek, + .wrapped_ek_len = wrapped_ek_len, + .key = key, + }; + int ret; + + if (!volume_uuid || !wrapped_ekwk || !wrapped_ekwk_len || !wrapped_ek || + !wrapped_ek_len || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_unwrap_file_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_unwrap_file_key); + +struct new_file_key_call { + const u8 *volume_uuid; + u32 protection_class; + u64 crypto_id; + u16 key_revision; + struct apple_sep_fv_new_file_key *key; +}; + +static int call_new_file_key(const struct sep_fv_ops *ops, void *argument) +{ + struct new_file_key_call *call = argument; + + return ops->new_file_key(apple_sep_fv_context, call->volume_uuid, + call->protection_class, call->crypto_id, + call->key_revision, call->key); +} + +int apple_sep_fv_new_file_key_v2(const u8 volume_uuid[16], + u32 protection_class, + u64 crypto_id, u16 key_revision, + struct apple_sep_fv_new_file_key *key) +{ + struct new_file_key_call call = { + .volume_uuid = volume_uuid, + .protection_class = protection_class, + .crypto_id = crypto_id, + .key_revision = key_revision, + .key = key, + }; + int ret; + + if (!volume_uuid || !crypto_id || !key_revision || !key) + return -EINVAL; + memzero_explicit(key, sizeof(*key)); + ret = apple_sep_fv_call(call_new_file_key, &call); + if (ret) + memzero_explicit(key, sizeof(*key)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_sep_fv_new_file_key_v2); diff --git a/drivers/soc/apple/hwrng.rs b/drivers/soc/apple/hwrng.rs new file mode 100644 index 00000000000000..92f3ac7043a788 --- /dev/null +++ b/drivers/soc/apple/hwrng.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Ownership of the hwrng-core registration. + +use kernel::prelude::*; + +pub(crate) type ReadFn = + unsafe extern "C" fn(ctx: *mut c_void, data: *mut c_void, max: usize, wait: bool) -> c_int; + +extern "C" { + fn sep_hwrng_alloc() -> *mut c_void; + fn sep_hwrng_free(mem: *mut c_void); + fn sep_hwrng_register( + mem: *mut c_void, + name: *const c_char, + quality: c_ushort, + ctx: *mut c_void, + read: Option, + ) -> c_int; + fn sep_hwrng_unregister(mem: *mut c_void); +} + +pub(crate) const QUALITY: c_ushort = 1024; + +pub(crate) const NAME: &CStr = c"apple-sep"; + +pub(crate) struct HwRngHandle { + mem: *mut c_void, + registered: bool, +} + +// SAFETY: `mem` is a plain heap allocation with no thread affinity; all access +// goes through the C shim, which locks inside the hwrng core. +unsafe impl Send for HwRngHandle {} + +impl HwRngHandle { + pub(crate) fn new() -> Result { + // SAFETY: no preconditions; returns NULL on allocation failure. + let mem = unsafe { sep_hwrng_alloc() }; + if mem.is_null() { + return Err(ENOMEM); + } + Ok(HwRngHandle { + mem, + registered: false, + }) + } + + /// # Safety + /// `ctx` must stay valid and safe to pass to `read` until [`Self::unregister`] + /// returns or this handle is dropped. + pub(crate) unsafe fn register(&mut self, ctx: *mut c_void, read: ReadFn) -> Result<()> { + if self.registered { + return Err(EBUSY); + } + // SAFETY: `mem` is live, `NAME` is a static NUL-terminated string, and + // the caller guarantees `ctx` outlives the registration. + let ret = + unsafe { sep_hwrng_register(self.mem, NAME.as_char_ptr(), QUALITY, ctx, Some(read)) }; + kernel::error::to_result(ret)?; + self.registered = true; + Ok(()) + } + + pub(crate) fn unregister(&mut self) { + if self.registered { + self.registered = false; + // SAFETY: `mem` is live and was registered. + unsafe { sep_hwrng_unregister(self.mem) }; + } + } +} + +impl Drop for HwRngHandle { + fn drop(&mut self) { + self.unregister(); + // SAFETY: `mem` is live and now unregistered, so nothing else refers to it. + unsafe { sep_hwrng_free(self.mem) }; + } +} diff --git a/drivers/soc/apple/hwrng_shim.c b/drivers/soc/apple/hwrng_shim.c new file mode 100644 index 00000000000000..e0507f935d0f76 --- /dev/null +++ b/drivers/soc/apple/hwrng_shim.c @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * hwrng core shim: struct hwrng lives here where the C compiler owns its + * layout, and Rust sees an opaque pointer plus four entry points. + */ + +#include +#include +#include + +#include "shim.h" + +struct sep_hwrng { + struct hwrng rng; + /* Opaque Rust-side context, handed back to the read callback. */ + void *ctx; + int (*read)(void *ctx, void *data, size_t max, bool wait); +}; + +static int sep_hwrng_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct sep_hwrng *h = + container_of(rng, struct sep_hwrng, rng); + + return h->read(h->ctx, data, max, wait); +} + +void *sep_hwrng_alloc(void) +{ + return kzalloc(sizeof(struct sep_hwrng), GFP_KERNEL); +} + +/* Must not be called while registered. */ +void sep_hwrng_free(void *mem) +{ + kfree(mem); +} + +/* + * @name must outlive the registration; the Rust side passes a &'static CStr. + * @quality is bits of entropy per 1024 bits of input; 0 takes the core default, + * which is also 1024. + */ +int sep_hwrng_register(void *mem, const char *name, + unsigned short quality, void *ctx, + int (*read)(void *ctx, void *data, size_t max, + bool wait)) +{ + struct sep_hwrng *h = mem; + + h->ctx = ctx; + h->read = read; + h->rng.name = name; + h->rng.read = sep_hwrng_read; + h->rng.quality = quality; + + return hwrng_register(&h->rng); +} + +/* + * Blocks until the core is done with the device, so any read in flight must + * return promptly: the Rust side sets its shutdown flag and wakes its waiters + * before calling this. + */ +void sep_hwrng_unregister(void *mem) +{ + struct sep_hwrng *h = mem; + + hwrng_unregister(&h->rng); +} diff --git a/drivers/soc/apple/image.rs b/drivers/soc/apple/image.rs new file mode 100644 index 00000000000000..8d4889268179d8 --- /dev/null +++ b/drivers/soc/apple/image.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Request and response images for the key-store endpoint. + +use kernel::prelude::*; + +extern "C" { + fn sep_sha256( + a: *const c_void, + alen: usize, + b: *const c_void, + blen: usize, + out: *mut u8, + ) -> c_int; +} + +pub(crate) const HEADER_SIZE: u32 = 0x50; + +pub(crate) const HEADER_WIRE: usize = 0x54; + +const DIGEST_OFF: usize = 0x00; +const DIGEST_LEN: usize = 16; + +const DIGEST_FROM: usize = 0x10; + +pub(crate) const SHA256_LEN: usize = 32; + +pub(crate) fn sha256(bytes: &[u8]) -> Result<[u8; SHA256_LEN]> { + let mut digest = [0u8; SHA256_LEN]; + // SAFETY: `bytes` is a live slice for the duration of the call and `digest` + // is exactly the 32 bytes the shim writes. The shim reads the second + // segment only when its length is nonzero, and it is zero here. + let rc = unsafe { + sep_sha256( + bytes.as_ptr().cast(), + bytes.len(), + core::ptr::null(), + 0, + digest.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok(digest) +} + +const OFF_VERSION: usize = 0x10; +const OFF_TIMESTAMP: usize = 0x14; +const OFF_TRAILER: usize = 0x48; + +static_assert!(OFF_VERSION == DIGEST_OFF + DIGEST_LEN); +static_assert!(OFF_TIMESTAMP == OFF_VERSION + 4); +static_assert!(OFF_TRAILER == OFF_TIMESTAMP + 8 + 4 + 8 + 8 + 4 + 20); +static_assert!(OFF_TRAILER + 8 == HEADER_SIZE as usize); +static_assert!(HEADER_WIRE == 4 + HEADER_SIZE as usize); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Version { + V1, +} + +impl Version { + pub(crate) const fn wire(self) -> u32 { + match self { + Version::V1 => 1, + } + } + + const fn digest_end(self) -> usize { + match self { + Version::V1 => OFF_TRAILER, + } + } +} + +pub(crate) struct Body { + bytes: KVec, +} + +pub(crate) fn wipe(bytes: &mut [u8]) { + for byte in bytes { + // SAFETY: `byte` is uniquely borrowed and valid. A volatile write keeps + // the compiler from eliding destruction of request secrets. + unsafe { core::ptr::write_volatile(byte, 0) }; + } +} + +impl Body { + pub(crate) fn new() -> Body { + Body { bytes: KVec::new() } + } + + pub(crate) fn put_u32(&mut self, v: u32) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_i32(&mut self, v: i32) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_u64(&mut self, v: u64) -> Result<()> { + self.bytes.extend_from_slice(&v.to_le_bytes(), GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn put_blob(&mut self, bytes: &[u8]) -> Result<()> { + let len = u32::try_from(bytes.len()).map_err(|_| EINVAL)?; + self.put_u32(len)?; + self.bytes.extend_from_slice(bytes, GFP_KERNEL)?; + let pad = bytes.len().wrapping_neg() % 4; + for _ in 0..pad { + self.bytes.push(0, GFP_KERNEL)?; + } + Ok(()) + } + + fn as_slice(&self) -> &[u8] { + &self.bytes + } +} + +impl Drop for Body { + fn drop(&mut self) { + wipe(&mut self.bytes); + } +} + +const fn pad_of(len: usize) -> usize { + len.wrapping_neg() % 4 +} +static_assert!(pad_of(0) == 0); +static_assert!(pad_of(1) == 3); +static_assert!(pad_of(2) == 2); +static_assert!(pad_of(3) == 1); +static_assert!(pad_of(4) == 0); + +pub(crate) struct RequestImage { + bytes: KVec, +} + +impl RequestImage { + pub(crate) fn as_slice(&self) -> &[u8] { + &self.bytes + } + + pub(crate) fn len(&self) -> usize { + self.bytes.len() + } +} + +impl Drop for RequestImage { + fn drop(&mut self) { + wipe(&mut self.bytes); + } +} + +pub(crate) fn build_request( + version: Version, + timestamp_us: u64, + body: &Body, +) -> Result { + let mut image = RequestImage { bytes: KVec::new() }; + let bytes = &mut image.bytes; + bytes.extend_from_slice(&HEADER_SIZE.to_le_bytes(), GFP_KERNEL)?; + + let header_at = bytes.len(); + for _ in 0..HEADER_SIZE { + bytes.push(0, GFP_KERNEL)?; + } + + let put = |bytes: &mut KVec, off: usize, src: &[u8]| { + bytes[header_at + off..header_at + off + src.len()].copy_from_slice(src); + }; + put(bytes, OFF_VERSION, &version.wire().to_le_bytes()); + put(bytes, OFF_TIMESTAMP, ×tamp_us.to_le_bytes()); + // Flags, reserved, proc_id, pid, cdhash and trailer stay zero; the enclave accepts that. + + bytes.extend_from_slice(body.as_slice(), GFP_KERNEL)?; + + static_assert!(OFF_VERSION == DIGEST_FROM); + let end = version.digest_end(); + let head = &bytes[header_at + DIGEST_FROM..header_at + end]; + let tail = &bytes[header_at + HEADER_SIZE as usize..]; + + let mut digest = [0u8; SHA256_LEN]; + // SAFETY: `head` and `tail` are live slices of `bytes` for the duration of + // the call, and `digest` is exactly the 32 bytes the shim writes. The shim + // reads only the two segments it is given and writes only the output. + let rc = unsafe { + sep_sha256( + head.as_ptr().cast(), + head.len(), + tail.as_ptr().cast(), + tail.len(), + digest.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + + bytes[header_at + DIGEST_OFF..header_at + DIGEST_OFF + DIGEST_LEN] + .copy_from_slice(&digest[..DIGEST_LEN]); + + Ok(image) +} + +pub(crate) struct ResponseImage<'a> { + pub(crate) body: &'a [u8], +} + +pub(crate) fn parse_response(bytes: &[u8]) -> Result> { + if bytes.len() < 4 { + return Err(EINVAL); + } + let header_size = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + + let header = header_size as usize; + if header < OFF_VERSION + 4 { + return Err(EINVAL); + } + let body_at = 4usize.checked_add(header).ok_or(EINVAL)?; + if bytes.len() < body_at { + return Err(EINVAL); + } + + Ok(ResponseImage { + body: &bytes[body_at..], + }) +} + +pub(crate) fn read_blob(body: &[u8], off: usize) -> Option<(&[u8], usize)> { + let len_end = off.checked_add(4)?; + if body.len() < len_end { + return None; + } + let len = u32::from_le_bytes([body[off], body[off + 1], body[off + 2], body[off + 3]]) as usize; + let end = len_end.checked_add(len)?; + if body.len() < end { + return None; + } + let next = end.checked_add(pad_of(len))?; + if body.get(end..next)?.iter().any(|byte| *byte != 0) { + return None; + } + Some((&body[len_end..end], next)) +} + +pub(crate) fn operation_status(body: &[u8]) -> Option { + if body.len() < 4 { + return None; + } + Some(i32::from_le_bytes([body[0], body[1], body[2], body[3]])) +} diff --git a/drivers/soc/apple/keybag.rs b/drivers/soc/apple/keybag.rs new file mode 100644 index 00000000000000..f97f6088217611 --- /dev/null +++ b/drivers/soc/apple/keybag.rs @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Host-side persistence for the key bag. +//! +//! `0x02` copies the bag out enclave-wrapped, `0x03` reloads it. NEVER create a +//! second bag while a stored blob exists — it orphans the previous one in the +//! enclave with no recovery; [`NoStoredKeyBag`] enforces this by type. + +use crate::shim; +use crate::store::crc16_ccitt_false; +use kernel::prelude::*; + +pub(crate) const KEYBAG_PATH: &CStr = c"/var/lib/aurora-sep-keybag.bin"; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Slot { + Identity, +} + +impl Slot { + pub(crate) fn path(self) -> &'static CStr { + match self { + Slot::Identity => KEYBAG_PATH, + } + } +} + +const MAGIC: [u8; 16] = *b"APPLE-SEP-KBAG01"; +/// v2 stores the bag secret; a v1 record is refused, not reinterpreted. +const VERSION: u32 = 2; + +const STATE_COMMITTED: u32 = 1; +/// Create refused by the enclave: nothing exists, retry allowed (unlike an +/// intent record, where a bag may exist unnamed — block retry). +const STATE_REFUSED: u32 = 2; +/// UUID field is the bag's own, read back via `0x06`; state 1 holds the +/// host-generated one, and for an identity bag these differ. +const STATE_COMMITTED_BAG_UUID: u32 = 3; + +/// Only an `AsGenerated` UUID may be repaired on a mismatch; adopting one for a +/// read-back record would point it at a different bag and strand the real one. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum UuidProvenance { + AsGenerated, + ReadBackFromBag, +} + +impl UuidProvenance { + fn state(self) -> u32 { + match self { + Self::AsGenerated => STATE_COMMITTED, + Self::ReadBackFromBag => STATE_COMMITTED_BAG_UUID, + } + } + + fn from_state(state: u32) -> Option { + match state { + STATE_COMMITTED => Some(Self::AsGenerated), + STATE_COMMITTED_BAG_UUID => Some(Self::ReadBackFromBag), + _ => None, + } + } +} + +const OFF_VERSION: usize = 0x10; +const OFF_STATE: usize = 0x14; +const OFF_LEN: usize = 0x18; +const OFF_CRC: usize = 0x1c; +const OFF_UUID: usize = 0x20; +const OFF_SECRET_LEN: usize = 0x30; +const HEADER: usize = 0x34; + +static_assert!(OFF_VERSION == MAGIC.len()); +static_assert!(OFF_STATE == OFF_VERSION + 4); +static_assert!(OFF_LEN == OFF_STATE + 4); +static_assert!(OFF_CRC == OFF_LEN + 4); +static_assert!(OFF_UUID == OFF_CRC + 4); +static_assert!(OFF_SECRET_LEN == OFF_UUID + UUID_LEN); +static_assert!(HEADER == OFF_SECRET_LEN + 4); + +pub(crate) const UUID_LEN: usize = 16; + +const MAX_WRAPPED: usize = crate::store::MAX_VALUE; + +/// Proof the host holds no bag; create takes one by value, so it is unreachable +/// without it and unrepeatable with it. +pub(crate) struct NoStoredKeyBag { + slot: Slot, +} + +impl NoStoredKeyBag { + pub(crate) fn slot(&self) -> Slot { + self.slot + } +} + +pub(crate) struct StoredKeyBag { + wrapped: KVec, + uuid: [u8; UUID_LEN], + secret: crate::Secret, + provenance: UuidProvenance, +} + +impl StoredKeyBag { + pub(crate) fn wrapped(&self) -> &[u8] { + &self.wrapped + } + + pub(crate) fn uuid(&self) -> &[u8; UUID_LEN] { + &self.uuid + } + + pub(crate) fn uuid_provenance(&self) -> UuidProvenance { + self.provenance + } + + pub(crate) fn secret(&self) -> &[u8] { + &self.secret + } +} + +pub(crate) enum State { + Present(StoredKeyBag), + Absent(NoStoredKeyBag), +} + +/// CRC preimage; this byte order is the on-disk contract — changing it breaks +/// records written by an earlier build. +fn checksum_input( + state: u32, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result { + let mut v = KVec::new(); + v.extend_from_slice(&VERSION.to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&state.to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&(wrapped.len() as u32).to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(&(secret.len() as u32).to_le_bytes(), GFP_KERNEL)?; + v.extend_from_slice(uuid, GFP_KERNEL)?; + v.extend_from_slice(wrapped, GFP_KERNEL)?; + v.extend_from_slice(secret, GFP_KERNEL)?; + Ok(crate::Secret(v)) +} + +/// Absent only when the file is missing; every other unreadable state errors — +/// "cannot tell" must never read as "nothing here", which would orphan a bag. +pub(crate) fn read(slot: Slot) -> Result { + let file = match shim::StoreFile::open_readonly(slot.path()) { + Ok(f) => f, + Err(e) if e == ENOENT => { + return Ok(State::Absent(NoStoredKeyBag { slot })); + } + Err(e) => return Err(e), + }; + + let size = file.size()?; + if size < HEADER as u64 { + return Err(EINVAL); + } + + let mut head = [0u8; HEADER]; + file.read_exact(0, &mut head)?; + if head[..MAGIC.len()] != MAGIC { + return Err(EINVAL); + } + let version = u32::from_le_bytes([ + head[OFF_VERSION], + head[OFF_VERSION + 1], + head[OFF_VERSION + 2], + head[OFF_VERSION + 3], + ]); + if version != VERSION { + return Err(ENOTSYNC); + } + let state = u32::from_le_bytes([ + head[OFF_STATE], + head[OFF_STATE + 1], + head[OFF_STATE + 2], + head[OFF_STATE + 3], + ]); + if state == STATE_REFUSED { + return Ok(State::Absent(NoStoredKeyBag { slot })); + } + let Some(provenance) = UuidProvenance::from_state(state) else { + // Intent or unknown state: a bag may exist unnamed; creating again would + // orphan it permanently. + return Err(EEXIST); + }; + + let len = u32::from_le_bytes([ + head[OFF_LEN], + head[OFF_LEN + 1], + head[OFF_LEN + 2], + head[OFF_LEN + 3], + ]) as usize; + if len == 0 || len > MAX_WRAPPED || size < (HEADER + len) as u64 { + return Err(EINVAL); + } + let crc = u16::from_le_bytes([head[OFF_CRC], head[OFF_CRC + 1]]); + + let mut uuid = [0u8; UUID_LEN]; + uuid.copy_from_slice(&head[OFF_UUID..OFF_UUID + UUID_LEN]); + + let secret_len = u32::from_le_bytes([ + head[OFF_SECRET_LEN], + head[OFF_SECRET_LEN + 1], + head[OFF_SECRET_LEN + 2], + head[OFF_SECRET_LEN + 3], + ]) as usize; + if secret_len > MAX_WRAPPED || size < (HEADER + len + secret_len) as u64 { + return Err(EINVAL); + } + + let mut wrapped = KVec::with_capacity(len, GFP_KERNEL)?; + wrapped.resize(len, 0, GFP_KERNEL)?; + file.read_exact(HEADER as u64, &mut wrapped)?; + + let mut secret_bytes = KVec::with_capacity(secret_len, GFP_KERNEL)?; + secret_bytes.resize(secret_len, 0, GFP_KERNEL)?; + if secret_len > 0 { + file.read_exact((HEADER + len) as u64, &mut secret_bytes)?; + } + + if size != (HEADER + len + secret_len) as u64 { + return Err(EINVAL); + } + let secret = crate::Secret(secret_bytes); + + if crc16_ccitt_false(&checksum_input(state, &wrapped, &uuid, &secret)?) != crc { + return Err(EINVAL); + } + + Ok(State::Present(StoredKeyBag { + wrapped, + uuid, + secret, + provenance, + })) +} + +pub(crate) fn write_intent(slot: Slot) -> Result<()> { + write_record(slot, 0, &[], &[0; UUID_LEN], &[]) +} + +pub(crate) fn mark_refused(slot: Slot) -> Result<()> { + write_record(slot, STATE_REFUSED, &[], &[0; UUID_LEN], &[]) +} + +pub(crate) fn write_bag_uuid( + slot: Slot, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + if wrapped.is_empty() || wrapped.len() > MAX_WRAPPED || secret.len() > MAX_WRAPPED { + return Err(EINVAL); + } + write_record( + slot, + UuidProvenance::ReadBackFromBag.state(), + wrapped, + uuid, + secret, + ) +} + +/// Replaces only the wrapped blob. The enclave ratchets bag material while a bag +/// is active, so this snapshot must be re-taken at the catacomb-save commit or +/// the enclave answers a later restore empty. +pub(crate) fn replace_wrapped( + slot: Slot, + fresh: &[u8], + snapshot_uuid: &[u8; UUID_LEN], +) -> Result<()> { + if fresh.is_empty() || fresh.len() > MAX_WRAPPED { + return Err(EINVAL); + } + let stored = match read(slot)? { + State::Present(stored) => stored, + State::Absent(_) => return Err(ENOENT), + }; + if stored.uuid() != snapshot_uuid { + // A snapshot of a different bag would strand the stored one. + return Err(EPERM); + } + write_record( + slot, + stored.uuid_provenance().state(), + fresh, + stored.uuid(), + stored.secret(), + ) +} + +fn write_record( + slot: Slot, + state: u32, + wrapped: &[u8], + uuid: &[u8; UUID_LEN], + secret: &[u8], +) -> Result<()> { + let crc = crc16_ccitt_false(&checksum_input(state, wrapped, uuid, secret)?); + + let mut head = [0u8; HEADER]; + head[..MAGIC.len()].copy_from_slice(&MAGIC); + head[OFF_VERSION..OFF_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + head[OFF_STATE..OFF_STATE + 4].copy_from_slice(&state.to_le_bytes()); + head[OFF_LEN..OFF_LEN + 4].copy_from_slice(&(wrapped.len() as u32).to_le_bytes()); + head[OFF_CRC..OFF_CRC + 2].copy_from_slice(&crc.to_le_bytes()); + head[OFF_UUID..OFF_UUID + UUID_LEN].copy_from_slice(uuid); + head[OFF_SECRET_LEN..OFF_SECRET_LEN + 4].copy_from_slice(&(secret.len() as u32).to_le_bytes()); + + let file = shim::StoreFile::open_trunc(slot.path())?; + // Body first, then the header that vouches for it: a crash between leaves a + // bad checksum, which `read` refuses — the safe side. + if !wrapped.is_empty() { + file.write_all(HEADER as u64, wrapped)?; + } + if !secret.is_empty() { + file.write_all((HEADER + wrapped.len()) as u64, secret)?; + } + file.write_all(0, &head)?; + file.sync() +} diff --git a/drivers/soc/apple/p256_shim.c b/drivers/soc/apple/p256_shim.c new file mode 100644 index 00000000000000..bcf1323e3c58f2 --- /dev/null +++ b/drivers/soc/apple/p256_shim.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — P-256 ECDH sender shim for the ref-key ECIES seal. + * + * The ref-key (op 0x22) SE-seal is an ECIES scheme: the AP encrypts to the + * ref-key's public point and the enclave decrypts (op "od"). The enclave holds + * the recipient private key and never exposes it; the encrypt half touches only + * the public key, so it runs here in the kernel with no secret crossing into + * the SEP request but the (public) ephemeral point. This shim does the sender's + * key agreement: generate an ephemeral P-256 keypair, return its public point + * for the envelope, and compute the ECDH shared secret against the recipient's. + * + * Byte order: the kernel's `ecdh-nist-p256` kpp is big-endian throughout + * (ecdh_set_secret decodes the key via ecc_digits_from_bytes; ecc_swap_digits + * reads/writes coordinates as __be64). So the private key, both public points + * (X||Y), and the shared secret are plain big-endian byte strings, matching the + * ECIES/SEP convention: this shim does no byte swapping. It only marshals + * through kmalloc'd buffers, since a scatterlist entry must live in the linear + * map and the caller's may not. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +#define P256_COORD_LEN 32 +#define P256_POINT_LEN 64 /* X || Y, big-endian, no 0x04 prefix */ + +/* + * Core key agreement. `priv_be` is a 32-byte big-endian private scalar, or NULL + * to generate a fresh ephemeral key. `pub_be_out` receives the 64-byte + * big-endian public point (X||Y), `shared_be_out` the 32-byte big-endian shared + * secret (X coordinate of priv*peer). Scatterlist buffers are kmalloc'd since + * sg_init_one requires the linear map. + */ +static int p256_agree(const u8 *priv_be, const u8 *peer_pub_be, + u8 *pub_be_out, u8 *shared_be_out) +{ + struct crypto_kpp *tfm; + struct kpp_request *req; + struct ecdh params; + struct scatterlist src, dst; + DECLARE_CRYPTO_WAIT(wait); + unsigned int enc_len; + char *enc = NULL; + u8 *pub_buf = NULL; + u8 *peer_buf = NULL; + u8 *shared_buf = NULL; + int rc; + + tfm = crypto_alloc_kpp("ecdh-nist-p256", 0, 0); + if (IS_ERR(tfm)) { + rc = PTR_ERR(tfm); + pr_err("apple_sep p256: alloc_kpp rc=%d\n", rc); + return rc; + } + + memset(¶ms, 0, sizeof(params)); + if (priv_be) { + params.key = (void *)priv_be; /* big-endian, as the kpp wants */ + params.key_size = P256_COORD_LEN; + } else { + params.key = NULL; + params.key_size = 0; /* kpp generates a valid random scalar */ + } + + enc_len = crypto_ecdh_key_len(¶ms); + enc = kmalloc(enc_len, GFP_KERNEL); + if (!enc) { + rc = -ENOMEM; + goto out_tfm; + } + rc = crypto_ecdh_encode_key(enc, enc_len, ¶ms); + if (rc) { + pr_err("apple_sep p256: encode_key rc=%d\n", rc); + goto out_enc; + } + rc = crypto_kpp_set_secret(tfm, enc, enc_len); + if (rc) { + pr_err("apple_sep p256: set_secret rc=%d\n", rc); + goto out_enc; + } + + req = kpp_request_alloc(tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto out_enc; + } + + pub_buf = kmalloc(P256_POINT_LEN, GFP_KERNEL); + if (!pub_buf) { + rc = -ENOMEM; + goto out_req; + } + sg_init_one(&dst, pub_buf, P256_POINT_LEN); + kpp_request_set_input(req, NULL, 0); + kpp_request_set_output(req, &dst, P256_POINT_LEN); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + rc = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait); + if (rc) { + pr_err("apple_sep p256: generate_public_key rc=%d\n", rc); + goto out_pub; + } + memcpy(pub_be_out, pub_buf, P256_POINT_LEN); + + peer_buf = kmalloc(P256_POINT_LEN, GFP_KERNEL); + shared_buf = kmalloc(P256_COORD_LEN, GFP_KERNEL); + if (!peer_buf || !shared_buf) { + rc = -ENOMEM; + goto out_pub; + } + memcpy(peer_buf, peer_pub_be, P256_POINT_LEN); + sg_init_one(&src, peer_buf, P256_POINT_LEN); + sg_init_one(&dst, shared_buf, P256_COORD_LEN); + kpp_request_set_input(req, &src, P256_POINT_LEN); + kpp_request_set_output(req, &dst, P256_COORD_LEN); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + rc = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait); + if (rc) { + pr_err("apple_sep p256: compute_shared_secret rc=%d\n", rc); + goto out_pub; + } + memcpy(shared_be_out, shared_buf, P256_COORD_LEN); + rc = 0; + +out_pub: + if (shared_buf) { + memzero_explicit(shared_buf, P256_COORD_LEN); + kfree(shared_buf); + } + kfree(peer_buf); + kfree(pub_buf); +out_req: + kpp_request_free(req); +out_enc: + kfree_sensitive(enc); +out_tfm: + crypto_free_kpp(tfm); + return rc; +} + +/* + * ECIES sender key agreement with a fresh ephemeral key. `peer_pub_be` is the + * recipient's 64-byte big-endian public point (0x04 prefix already stripped by + * the caller). `eph_pub_be_out` receives the 64-byte big-endian ephemeral + * public point (caller prepends 0x04 for the envelope), `shared_be_out` the + * 32-byte big-endian shared secret for the KDF. + */ +int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, + void *shared_be_out) +{ + return p256_agree(NULL, peer_pub_be, eph_pub_be_out, shared_be_out); +} + diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs new file mode 100644 index 00000000000000..61faf6b88736ff --- /dev/null +++ b/drivers/soc/apple/profile.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Typed platform profiles for the SEP driver. +//! +//! Every SoC-specific fact the driver needs — the shared-memory capacity, the +//! boot handshake, the OS-identity source and the sensor transport — is named +//! here once. The driver reads addresses from the device tree, never scans for +//! them, and never patches properties at runtime. +//! +//! Two bring-up targets are modelled: +//! +//! * `T8103` / J313 (MacBook Air, M1): the host boots the SEP with the boot +//! endpoint handshake over a 0x30000 shared-memory window. +//! * `T6020` / J414s (MacBook Pro 14", M2 Pro): the driver does the warm +//! single-message registration over a 0x40000 window. + +// The boot handshake, identity source, sensor and DART fields are consumed by +// the boot-endpoint, identity and sensor-transport paths. +#![allow(dead_code)] + +use kernel::of; +use kernel::prelude::*; + +/// How the driver brings the shared-memory table to the SEP. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Bootstrap { + /// Cold boot: send TZ0, map the firmware reserved region and answer the + /// second boot acknowledgement with the firmware address and `SET_SHMEM`. + Boot, + /// Warm attach: send the single shared-memory registration message with the + /// table address and size. + WarmRegister, +} + +/// Where the 16-byte OS identity comes from. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum IdentitySource { + /// `/chosen/apfs-preboot-uuid`, the value the firmware handed to this boot + /// and the boot chain forwards. A malformed or absent property is refused; + /// the identity is never invented. + Chosen, + /// A UUID provisioned in the Linux host-state store on first bring-up. + HostPersisted, +} + +/// SPI mode as the device tree spells it. Mode 1 is CPOL=0/CPHA=1 (`spi-cpha`); +/// mode 2 is CPOL=1/CPHA=0 (`spi-cpol`). +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum SpiMode { + Mode1, + Mode2, +} + +impl SpiMode { + /// The value the SPI shim programs into `spi_setup()`, using the kernel's + /// `SPI_CPHA`/`SPI_CPOL` bits. + pub(crate) const fn wire(self) -> u32 { + match self { + SpiMode::Mode1 => 1, // SPI_CPHA + SpiMode::Mode2 => 2, // SPI_CPOL + } + } + + pub(crate) const fn name(self) -> &'static CStr { + match self { + SpiMode::Mode1 => c"mode 1 (CPOL=0 CPHA=1)", + SpiMode::Mode2 => c"mode 2 (CPOL=1 CPHA=0)", + } + } +} + +pub(crate) struct SensorProfile { + /// SPI controller register base. Diagnostics only: the device tree is + /// authoritative and the driver never enables or creates the bus. + pub(crate) controller_base: u64, + pub(crate) chip_select: u32, + pub(crate) max_hz: u32, + /// Chip-select setup and hold in nanoseconds. + pub(crate) cs_setup_ns: u32, + pub(crate) cs_hold_ns: u32, + pub(crate) mode: SpiMode, + pub(crate) expected_id: u16, + /// Capture traffic stays disabled until a qualified DMA transport exists; + /// the PIO path is status-only. + pub(crate) capture_qualified: bool, +} + +/// Identity keybag `CREATE_KEYBAG` field encoding. The strict T8103 enclave +/// reads the request's first word as the bag type and rejects the value the +/// lenient T6020 enclave accepts there; macOS carries the type in the third +/// word instead. Kept per-SoC so the strict path is correct without disturbing +/// the proven T6020 encoding (hardware-verified for enrol, match, and reboot). +pub(crate) struct KeybagCreate { + /// First word: request variant, echoed back in the reply. + pub(crate) variant: u32, + /// Third word: bag type. Identity is `0x20000` on the strict path; `0` on + /// T6020, which distinguishes the bag by the variant word instead. + pub(crate) bag_type: u32, + /// Fourth word: create argument / parent handle. + pub(crate) arg: i32, +} + +pub(crate) struct PlatformProfile { + pub(crate) name: &'static str, + /// Capacity of the boot shared-memory window. The layout is still checked + /// against the manifests; this only bounds the allocation. + pub(crate) shmem_capacity: usize, + /// Fourcc of the first shared-memory item. The two paths spell it + /// differently, so each profile carries its own spelling. + pub(crate) shmem_first_item: &'static [u8; 4], + pub(crate) bootstrap: Bootstrap, + pub(crate) identity: IdentitySource, + pub(crate) sensor: SensorProfile, + /// Require a static `apple,dma-range` on the SEP DART. The T6020 SEP only + /// accepts IOVAs below 4 GiB; T8103 works with the stock DART aperture. + pub(crate) dart_range_required: bool, + /// Reserved-memory region holding the SEP firmware image (cold-boot path). + pub(crate) firmware_region: &'static CStr, + /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). + pub(crate) keybag_create: KeybagCreate, +} + +const T8103: PlatformProfile = PlatformProfile { + name: "T8103/J313", + shmem_capacity: 0x3_0000, + shmem_first_item: b"CNIP", + bootstrap: Bootstrap::Boot, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: false, + firmware_region: c"sepfw", + // Strict enclave: the type goes in the third word; the first word is 0. + keybag_create: KeybagCreate { + variant: 0, + bag_type: 0x20000, + arg: 0, + }, +}; + +const T6020: PlatformProfile = PlatformProfile { + name: "T6020/J414s", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::HostPersisted, + sensor: SensorProfile { + controller_base: 0x3_9b10_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode2, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + firmware_region: c"sepfw", + // Proven encoding: the lenient enclave takes the variant in the first word. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, +}; + +static_assert!(T8103.shmem_capacity == 0x30000); +static_assert!(T6020.shmem_capacity == 0x40000); +static_assert!(T8103.sensor.controller_base == 0x235108000); +static_assert!(T6020.sensor.controller_base == 0x39b108000); +static_assert!(T8103.sensor.expected_id == T6020.sensor.expected_id); + +/// Whether the machine root declares `compatible`. +/// +/// The root property is a NUL-separated list, so whole entries are compared +/// instead of substrings: `apple,t8103` cannot match a longer unrelated value. +fn machine_has(compatible: &[u8]) -> bool { + let Some(root) = of::root() else { + return false; + }; + let Ok(list) = root.get_property::>(c"compatible") else { + return false; + }; + + let mut at = 0usize; + while at < list.len() { + let end = match list[at..].iter().position(|&byte| byte == 0) { + Some(offset) => at + offset, + None => list.len(), + }; + if &list[at..end] == compatible { + return true; + } + at = end + 1; + } + + false +} + +/// Select the profile for the running machine. An unsupported SoC is refused +/// rather than guessed at, so the driver cannot run a handshake with the wrong +/// geometry. +pub(crate) fn detect() -> Result<&'static PlatformProfile> { + if machine_has(b"apple,t8103") { + return Ok(&T8103); + } + if machine_has(b"apple,t6020") { + return Ok(&T6020); + } + Err(ENODEV) +} diff --git a/drivers/soc/apple/proto.rs b/drivers/soc/apple/proto.rs new file mode 100644 index 00000000000000..de2bf4580d84a5 --- /dev/null +++ b/drivers/soc/apple/proto.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +pub(crate) const EP_CONTROL: u8 = 0x00; +pub(crate) const EP_DISCOVER: u8 = 0xFD; +pub(crate) const EP_SHMEM: u8 = 0xFE; +pub(crate) const EP_BOOT: u8 = 0xFF; +pub(crate) const EP_XARM: u8 = 0x13; +pub(crate) const EP_SBIO: u8 = 0x08; +pub(crate) const EP_SCRD: u8 = 0x0a; +pub(crate) const EP_SKS: u8 = 0x12; + +pub(crate) const DISCOVER_TYPE_DESCRIPTOR: u8 = 0x00; +pub(crate) const DISCOVER_TYPE_CONFIG: u8 = 0x01; + +pub(crate) const MSG_TAG_SHIFT: u32 = 8; +pub(crate) const MSG_TYPE_SHIFT: u32 = 16; +pub(crate) const MSG_PARAM_SHIFT: u32 = 24; +pub(crate) const MSG_DATA_SHIFT: u32 = 32; + +// Boot-endpoint (EP_BOOT) message types for the cold-boot handshake: send TZ0, +// then hand over the firmware image (IMG4) and the shared-memory table. +pub(crate) const MSG_BOOT_TZ0: u64 = 0x5; +pub(crate) const MSG_BOOT_IMG4: u64 = 0x6; +pub(crate) const MSG_SET_SHMEM: u64 = 0x18; +pub(crate) const MSG_BOOT_TZ0_ACK1: u64 = 0x69; +pub(crate) const MSG_BOOT_TZ0_ACK2: u64 = 0xd2; +pub(crate) const MSG_BOOT_IMG4_ACK: u64 = 0x6a; + +// 4 KiB units even though CPU pages are 16 KiB +pub(crate) const IOVA_SHIFT: u32 = 12; + +#[derive(Clone, Copy)] +pub(crate) struct Fields { + pub(crate) ep: u8, + pub(crate) tag: u8, + pub(crate) ty: u8, + pub(crate) param: u8, + pub(crate) data_lo: u32, +} + +pub(crate) fn decode(msg: &Message) -> Fields { + Fields { + ep: msg.msg0 as u8, + tag: (msg.msg0 >> MSG_TAG_SHIFT) as u8, + ty: (msg.msg0 >> MSG_TYPE_SHIFT) as u8, + param: (msg.msg0 >> MSG_PARAM_SHIFT) as u8, + data_lo: (msg.msg0 >> MSG_DATA_SHIFT) as u32, + } +} + +pub(crate) const fn encode_registration_msg0(iova: u64, size: usize) -> u64 { + (EP_SHMEM as u64) + | (((size as u64) >> IOVA_SHIFT) << MSG_TYPE_SHIFT) + | ((iova >> IOVA_SHIFT) << MSG_DATA_SHIFT) +} + +static_assert!(encode_registration_msg0(0xBEE0_0000, 0x4_0000) == 0x000b_ee00_0040_00fe); + +pub(crate) fn shmem_registration(iova: u64, size: usize) -> Result { + let unit = 1u64 << IOVA_SHIFT; + + if size == 0 || (size as u64) & (unit - 1) != 0 { + return Err(EINVAL); + } + if iova & (unit - 1) != 0 { + return Err(EINVAL); + } + + let size_field = (size as u64) >> IOVA_SHIFT; + if size_field > 0xFF { + return Err(EINVAL); + } + + let iova_field = iova >> IOVA_SHIFT; + if iova_field > u64::from(u32::MAX) { + return Err(EINVAL); + } + + Ok(Message { + msg0: encode_registration_msg0(iova, size), + msg1: 0, + }) +} + +pub(crate) const CONTROL_REPLY_TYPE: u8 = 0x01; + +pub(crate) const TAG_ENTROPY: u8 = 0xE7; + +pub(crate) const CONTROL_TIMEOUT_MS: u32 = 2000; + +pub(crate) const TAG_POOL_FIRST: u8 = 0x01; +pub(crate) const TAG_POOL_LAST: u8 = 0x7e; + +static_assert!(TAG_POOL_FIRST > 0); +static_assert!(TAG_POOL_LAST < TAG_ENTROPY); + +pub(crate) struct ControlOp { + ty: u8, + param: u8, + data: u32, + expect_reply: bool, + timeout_ms: u32, + reserved_tag: bool, + name: &'static CStr, +} + +impl ControlOp { + pub(crate) fn expects_reply(&self) -> bool { + self.expect_reply + } + pub(crate) fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + pub(crate) fn uses_reserved_tag(&self) -> bool { + self.reserved_tag + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +pub(crate) fn op_nop(param: u8) -> ControlOp { + ControlOp { + ty: 0x00, + param, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name: c"NOP", + } +} + +pub(crate) fn op_security_mode() -> ControlOp { + ControlOp { + ty: 0x14, + param: 0x00, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name: c"SECMODE", + } +} + +// no op for control 0x18: it wedges the control endpoint + +pub(crate) fn op_get_entropy() -> ControlOp { + ControlOp { + ty: 0x36, + param: 0x00, + data: 0, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: true, + name: c"GET_ENTROPY", + } +} + +pub(crate) fn encode_control(op: &ControlOp, tag: u8) -> Message { + Message { + msg0: u64::from(EP_CONTROL) + | (u64::from(tag) << MSG_TAG_SHIFT) + | (u64::from(op.ty) << MSG_TYPE_SHIFT) + | (u64::from(op.param) << MSG_PARAM_SHIFT) + | (u64::from(op.data) << MSG_DATA_SHIFT), + msg1: 0, + } +} + +const OP_OOL_INBOUND_SIZE: u8 = 0x04; +const OP_OOL_INBOUND_ADDR: u8 = 0x02; +const OP_OOL_OUTBOUND_SIZE: u8 = 0x05; +const OP_OOL_OUTBOUND_ADDR: u8 = 0x03; + +fn op_ool(ty: u8, endpoint: u8, data: u32, name: &'static CStr) -> ControlOp { + ControlOp { + ty, + param: endpoint, + data, + expect_reply: true, + timeout_ms: CONTROL_TIMEOUT_MS, + reserved_tag: false, + name, + } +} + +pub(crate) fn op_ool_inbound_size(endpoint: u8, len: u32) -> ControlOp { + op_ool(OP_OOL_INBOUND_SIZE, endpoint, len, c"OOL_IN_SIZE") +} + +pub(crate) fn op_ool_inbound_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_OOL_INBOUND_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"OOL_IN_ADDR", + ) +} + +pub(crate) fn op_ool_outbound_size(endpoint: u8, len: u32) -> ControlOp { + op_ool(OP_OOL_OUTBOUND_SIZE, endpoint, len, c"OOL_OUT_SIZE") +} + +pub(crate) fn op_ool_outbound_addr(endpoint: u8, iova: u64) -> ControlOp { + op_ool( + OP_OOL_OUTBOUND_ADDR, + endpoint, + (iova >> IOVA_SHIFT) as u32, + c"OOL_OUT_ADDR", + ) +} + +pub(crate) struct FieldCursor<'a> { + body: &'a [u8], + at: usize, +} + +impl<'a> FieldCursor<'a> { + pub(crate) fn new(body: &'a [u8]) -> FieldCursor<'a> { + FieldCursor { body, at: 0 } + } + + pub(crate) fn i32(&mut self) -> Option { + let end = self.at.checked_add(4)?; + let v = i32::from_le_bytes(self.body.get(self.at..end)?.try_into().ok()?); + self.at = end; + Some(v) + } + + // blob = u32 len + bytes + zero pad to 4-byte boundary; 0x45 reply has two in a row + pub(crate) fn blob(&mut self) -> Option<&'a [u8]> { + let len_end = self.at.checked_add(4)?; + let len = u32::from_le_bytes(self.body.get(self.at..len_end)?.try_into().ok()?) as usize; + let end = len_end.checked_add(len)?; + let bytes = self.body.get(len_end..end)?; + let pad = len.wrapping_neg() % 4; + let next = end.checked_add(pad)?; + if self.body.get(end..next)?.iter().any(|byte| *byte != 0) { + return None; + } + self.at = next; + Some(bytes) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ControlReply { + pub(crate) tag: u8, + pub(crate) data_lo: u32, +} + +impl ControlReply { + pub(crate) fn from_message(msg: &Message) -> ControlReply { + let f = decode(msg); + ControlReply { + tag: f.tag, + data_lo: f.data_lo, + } + } +} diff --git a/drivers/soc/apple/refkey.rs b/drivers/soc/apple/refkey.rs new file mode 100644 index 00000000000000..d2ffe586594ea8 --- /dev/null +++ b/drivers/soc/apple/refkey.rs @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Machine ref-key: Secure-Enclave key sealing, signing, and attestation. +//! +//! A ref-key (op `0x22`) is an EC keypair the enclave keeps; the private half +//! never leaves the SEP. The host-side ECIES encrypt lives in [`refkey_seal`]; +//! every half that needs the private key runs in the enclave. + +use crate::{image, keybag, proto, refkey_seal, shim}; +use crate::{LockState, MachineRefKey, SepData, SksRequest}; +use kernel::prelude::*; + +pub(crate) const REFKEY_ENVELOPE_VERSION: u32 = 2; + +impl SepData { + fn sks_req_refkey(&self, keybag: i32, der_set: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(keybag)?; + body.put_u32(crate::refkey::REFKEY_ENVELOPE_VERSION)?; + body.put_blob(der_set)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = + crate::sks::encode_sks_perform_operation(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_PERFORM_OP_NAME, + msg, + img, + }) + } + + fn sks_create_refkey( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + + // Class 9 = non-extractable (private stays enclave-resident); key type 5 = P-256. + const PROTECTION_CLASS: u32 = 9; + const KEY_TYPE: u32 = 5; + + // Unlock the bag first; an unbound create is refused. + if let Some(healthy) = self.sks_health_check(c"ref-key create") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + secret, + healthy, + )); + } + + let create_der = { + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + if items.push((b"o", RV::Utf8(b"oc")), GFP_KERNEL).is_err() + || items + .push((b"bc", RV::Integer(PROTECTION_CLASS)), GFP_KERNEL) + .is_err() + || items + .push((b"kt", RV::Integer(KEY_TYPE)), GFP_KERNEL) + .is_err() + { + return None; + } + crate::der::encode_refkey_set(&items).ok()? + }; + let cout = self.sks_send(self.sks_req_refkey(handle.value(), &create_der))?; + if cout.reply.status != 0 { + return None; + } + let cbody = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &cout)?; + let mut cf = proto::FieldCursor::new(cbody); + let (Some(0), Some(cblob)) = (cf.i32(), cf.blob()) else { + return None; + }; + let mut refkey_blob: KVec = KVec::new(); + refkey_blob.extend_from_slice(cblob, GFP_KERNEL).ok()?; + Some(refkey_blob) + } + + fn refkey_pub(blob: &[u8]) -> Option<&[u8]> { + let rk_tlv = crate::der::refkey_find(blob, b"rk")?; + let pub_raw = + crate::der::refkey_find(rk_tlv, b"pub").and_then(crate::der::octet_string_body)?; + (pub_raw.len() == refkey_seal::POINT_LEN).then_some(pub_raw) + } + + pub(crate) fn sks_machine_refkey(&self, handle: crate::sks::KeyBagHandle, secret: &[u8]) { + const MACHINE_REFKEY_PATH: &CStr = c"/var/lib/aurora-sep-refkey.bin"; + + if self.machine_refkey.lock().is_some() { + return; + } + + if let Ok(f) = shim::StoreFile::open_readonly(MACHINE_REFKEY_PATH) { + let sz = f.size().unwrap_or(0); + if (1..=8192).contains(&sz) { + let mut blob: KVec = KVec::new(); + if blob.resize(sz as usize, 0u8, GFP_KERNEL).is_ok() + && f.read_exact(0, &mut blob).is_ok() + && self.cache_machine_refkey(blob) + { + return; + } + } + } + + let Some(blob) = self.sks_create_refkey(handle, secret) else { + dev_warn!( + self.dev, + "sks: could not create the machine ref-key; trusted-key sealing is unavailable this boot.\n" + ); + return; + }; + if let Ok(f) = shim::StoreFile::open(MACHINE_REFKEY_PATH) { + if f.write_all(0, &blob).is_ok() { + let _ = f.sync(); + } + } + let _ = self.cache_machine_refkey(blob); + } + + fn cache_machine_refkey(&self, blob: KVec) -> bool { + let Some(pubv) = Self::refkey_pub(&blob) else { + return false; + }; + let mut pub_raw: KVec = KVec::new(); + if pub_raw.extend_from_slice(pubv, GFP_KERNEL).is_err() { + return false; + } + *self.machine_refkey.lock() = Some(MachineRefKey { blob, pub_raw }); + true + } + + fn ensure_machine_refkey(&self) -> Result<()> { + if self.machine_refkey.lock().is_some() { + return Ok(()); + } + if !self.sks_ready() { + return Err(ENODEV); + } + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + self.sks_machine_refkey(handle, stored.secret()); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + if self.machine_refkey.lock().is_some() { + Ok(()) + } else { + Err(EIO) + } + } + + pub(crate) fn refkey_seal_trusted(&self, key: &[u8]) -> Result> { + self.ensure_machine_refkey()?; + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + refkey_seal::ecies_seal(&mk.pub_raw, key) + } + + pub(crate) fn refkey_unseal_trusted(&self, sealed: &[u8]) -> Result> { + self.ensure_machine_refkey()?; + let blob = { + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + let mut b: KVec = KVec::new(); + b.extend_from_slice(&mk.blob, GFP_KERNEL)?; + b + }; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if let Some(healthy) = self.sks_health_check(c"trusted-key unseal") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + let recovered = self.sks_refkey_unseal(handle, &blob, sealed); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + recovered.ok_or(EIO) + } + + /// Op `osgn`: ECDSA-P256 over `challenge` as the pre-computed digest. + fn sks_refkey_sign( + &self, + handle: crate::sks::KeyBagHandle, + refkey_blob: &[u8], + challenge: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + items.push((b"o", RV::Utf8(b"osgn")), GFP_KERNEL).ok()?; + items.push((b"d", RV::Octets(challenge)), GFP_KERNEL).ok()?; + items.push((b"rk", RV::Der(refkey_blob)), GFP_KERNEL).ok()?; + let der = crate::der::encode_refkey_set(&items).ok()?; + let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; + if out.reply.status != 0 { + dev_warn!( + self.dev, + "sks: ref-key attest sign failed (status {})\n", + out.reply.status + ); + return None; + } + let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; + let mut f = proto::FieldCursor::new(body); + let (Some(0), Some(sig)) = (f.i32(), f.blob()) else { + return None; + }; + // Enclave wraps the sig in OCTET STRING { SEQUENCE { r, s } }; strip to inner DER. + let der_sig = crate::der::octet_string_body(sig).unwrap_or(sig); + let mut owned: KVec = KVec::new(); + owned.extend_from_slice(der_sig, GFP_KERNEL).ok()?; + Some(owned) + } + + /// Attestation of key possession by signing proof; op `oa` (attestation-chained) + /// needs the SEP device attestation key, absent on a Linux-attached SEP. + pub(crate) fn refkey_attest_sign(&self, challenge: &[u8]) -> Result<(KVec, KVec)> { + self.ensure_machine_refkey()?; + let (blob, pubk) = { + let guard = self.machine_refkey.lock(); + let mk = guard.as_ref().ok_or(ENODEV)?; + let mut b: KVec = KVec::new(); + b.extend_from_slice(&mk.blob, GFP_KERNEL)?; + let mut p: KVec = KVec::new(); + p.extend_from_slice(&mk.pub_raw, GFP_KERNEL)?; + (b, p) + }; + let keybag::State::Present(stored) = keybag::read(keybag::Slot::Identity)? else { + return Err(ENODEV); + }; + let (handle, _uuid) = self.sks_recover(&stored).ok_or(EIO)?; + if let Some(healthy) = self.sks_health_check(c"ref-key attest") { + let _ = self.sks_send(self.sks_req_change_lock_state( + handle, + LockState::Unlocked, + stored.secret(), + healthy, + )); + } + let sig = self.sks_refkey_sign(handle, &blob, challenge); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + Ok((sig.ok_or(EIO)?, pubk)) + } + + /// Enclave ECIES-decrypt (o = "oecd"); takes the ephemeral from the front of `d`. + fn sks_refkey_unseal( + &self, + handle: crate::sks::KeyBagHandle, + refkey_blob: &[u8], + sealed: &[u8], + ) -> Option> { + use crate::der::RefKeyValue as RV; + let mut items: KVec<(&[u8], RV<'_>)> = KVec::new(); + items.push((b"o", RV::Utf8(b"oecd")), GFP_KERNEL).ok()?; + items.push((b"d", RV::Octets(sealed)), GFP_KERNEL).ok()?; + items.push((b"rk", RV::Der(refkey_blob)), GFP_KERNEL).ok()?; + let der = crate::der::encode_refkey_set(&items).ok()?; + let out = self.sks_send(self.sks_req_refkey(handle.value(), &der))?; + if out.reply.status != 0 { + dev_warn!( + self.dev, + "trusted-keys: ref-key unseal failed (status {})\n", + out.reply.status + ); + return None; + } + let body = self.sks_report_response(crate::sks::SKS_PERFORM_OP_NAME, &out)?; + let mut f = proto::FieldCursor::new(body); + match (f.i32(), f.blob()) { + (Some(0), Some(rec)) => { + let mut owned = KVec::new(); + owned.extend_from_slice(rec, GFP_KERNEL).ok()?; + Some(owned) + } + _ => None, + } + } +} diff --git a/drivers/soc/apple/refkey_seal.rs b/drivers/soc/apple/refkey_seal.rs new file mode 100644 index 00000000000000..f1d95935690b58 --- /dev/null +++ b/drivers/soc/apple/refkey_seal.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Ref-key (op 0x22) ECIES **encrypt** — the AP half of the SE seal. +//! +//! The SEP has no scalar×G, so it cannot make the sender ephemeral (in-SEP `ow` +//! returns -10); `od` imports the one this module produces, ECDHs it against the +//! in-SEP private key and AES-GCM-decrypts. The encrypt runs here by construction. + +use kernel::prelude::*; + +extern "C" { + fn sep_p256_sender( + peer_pub_be: *const c_void, + eph_pub_be_out: *mut u8, + shared_be_out: *mut u8, + ) -> c_int; + + fn sep_gcm( + encrypt: c_int, + key: *const c_void, + keylen: usize, + iv: *const c_void, + ivlen: usize, + aadlen: usize, + buf: *mut c_void, + buflen: usize, + datalen: usize, + ) -> c_int; +} + +// The enclave's SE seal is an ECIES scheme: +// ECDH P-256 (shared = X), X9.63-SHA256 KDF, KDF-derived key‖IV, AES-GCM. + +pub(crate) const TAG_LEN: usize = 16; +/// 16-byte IV (KDF-derived "variable IV"), not the 12-byte GCM default. +pub(crate) const IV_LEN: usize = 16; +pub(crate) const AES256_KEY_LEN: usize = 32; +pub(crate) const AES128_KEY_LEN: usize = 16; +/// Uncompressed P-256 point: `0x04 ‖ X ‖ Y`. +pub(crate) const POINT_LEN: usize = 65; +/// Raw `X ‖ Y` (no prefix), as the ECDH shim exchanges. +const COORDS_LEN: usize = 64; + +fn wipe(bytes: &mut [u8]) { + for b in bytes.iter_mut() { + // SAFETY: `b` is a valid, uniquely borrowed byte for this write. + unsafe { core::ptr::write_volatile(b, 0) }; + } +} + +fn p256_sender(peer_pub64: &[u8]) -> Result<([u8; COORDS_LEN], [u8; 32])> { + if peer_pub64.len() != COORDS_LEN { + return Err(EINVAL); + } + let mut eph = [0u8; COORDS_LEN]; + let mut shared = [0u8; 32]; + // SAFETY: `peer_pub64` is 64 live bytes; `eph`/`shared` are exactly the 64 + // and 32 bytes the shim writes. + let rc = unsafe { + sep_p256_sender( + peer_pub64.as_ptr().cast(), + eph.as_mut_ptr(), + shared.as_mut_ptr(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + Ok((eph, shared)) +} + +fn kdf_sha256_mode(z: &[u8], shared_info: &[u8], out_len: usize, mode: u8) -> Result> { + let mut out = KVec::new(); + let mut counter: u32 = 1; + while out.len() < out_len { + let ctr = counter.to_be_bytes(); + let mut buf = KVec::new(); + match mode { + 0 => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + 1 => { + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + 2 => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + } + 3 => { + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(&ctr, GFP_KERNEL)?; + } + _ => { + buf.extend_from_slice(z, GFP_KERNEL)?; + buf.extend_from_slice(shared_info, GFP_KERNEL)?; + } + } + let digest = crate::image::sha256(&buf)?; + wipe(&mut buf); + let take = core::cmp::min(digest.len(), out_len - out.len()); + out.extend_from_slice(&digest[..take], GFP_KERNEL)?; + counter = counter.checked_add(1).ok_or(EINVAL)?; + } + Ok(out) +} + +fn gcm_encrypt(key: &[u8], iv: &[u8], aad: &[u8], plaintext: &[u8]) -> Result> { + let mut buf = KVec::new(); + buf.extend_from_slice(aad, GFP_KERNEL)?; + buf.extend_from_slice(plaintext, GFP_KERNEL)?; + buf.resize(aad.len() + plaintext.len() + TAG_LEN, 0u8, GFP_KERNEL)?; + // SAFETY: `key`/`iv` are live; `buf` holds aad‖plaintext‖tag-space and its + // length is passed as both the buffer size and the extent to authenticate. + let rc = unsafe { + sep_gcm( + 1, + key.as_ptr().cast(), + key.len(), + iv.as_ptr().cast(), + iv.len(), + aad.len(), + buf.as_mut_ptr().cast(), + buf.len(), + plaintext.len(), + ) + }; + if rc != 0 { + return Err(Error::from_errno(rc)); + } + let mut out = KVec::new(); + out.extend_from_slice(&buf[aad.len()..], GFP_KERNEL)?; + Ok(out) +} + +pub(crate) struct EciesParts { + /// Ephemeral point `0x04 ‖ X ‖ Y`; also the KDF sharedInfo. + pub(crate) eph_point: KVec, + pub(crate) ciphertext: KVec, + pub(crate) tag: KVec, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn ecies_encrypt_parts( + recipient_pub: &[u8], + plaintext: &[u8], + key_len: usize, + variable_iv: bool, + iv_len: usize, + aad_eph: bool, + si_mode: u8, + kdf_mode: u8, +) -> Result { + if recipient_pub.len() != POINT_LEN || recipient_pub[0] != 0x04 { + return Err(EINVAL); + } + if key_len != AES256_KEY_LEN && key_len != AES128_KEY_LEN { + return Err(EINVAL); + } + if iv_len != 12 && iv_len != 16 { + return Err(EINVAL); + } + let (eph_coords, mut shared) = p256_sender(&recipient_pub[1..])?; + + let mut eph_point = KVec::new(); + eph_point.push(0x04, GFP_KERNEL)?; + eph_point.extend_from_slice(&eph_coords, GFP_KERNEL)?; + + let mut si = KVec::new(); + match si_mode { + 0 => si.extend_from_slice(&eph_point, GFP_KERNEL)?, + 1 => {} + 2 => si.extend_from_slice(&eph_point[1..], GFP_KERNEL)?, + 3 => si.extend_from_slice(&eph_point[1..33], GFP_KERNEL)?, + 4 => si.extend_from_slice(recipient_pub, GFP_KERNEL)?, + _ => { + si.extend_from_slice(&eph_point, GFP_KERNEL)?; + si.extend_from_slice(recipient_pub, GFP_KERNEL)?; + } + } + + let kdf_len = if variable_iv { + key_len + iv_len + } else { + key_len + }; + let mut km = kdf_sha256_mode(&shared, &si, kdf_len, kdf_mode)?; + wipe(&mut shared); + let mut iv = KVec::new(); + if variable_iv { + iv.extend_from_slice(&km[key_len..key_len + iv_len], GFP_KERNEL)?; + } else { + iv.resize(iv_len, 0u8, GFP_KERNEL)?; + } + let aad: &[u8] = if aad_eph { &eph_point } else { &[] }; + let ct_tag = gcm_encrypt(&km[..key_len], &iv, aad, plaintext); + wipe(&mut km); + let ct_tag = ct_tag?; + if ct_tag.len() != plaintext.len() + TAG_LEN { + return Err(EINVAL); + } + + let mut ciphertext = KVec::new(); + ciphertext.extend_from_slice(&ct_tag[..plaintext.len()], GFP_KERNEL)?; + let mut tag = KVec::new(); + tag.extend_from_slice(&ct_tag[plaintext.len()..], GFP_KERNEL)?; + + Ok(EciesParts { + eph_point, + ciphertext, + tag, + }) +} + +/// Returns the enclave-ready blob: `ephemeral_point ‖ ciphertext ‖ tag`. +pub(crate) fn ecies_seal(recipient_pub: &[u8], secret: &[u8]) -> Result> { + let parts = ecies_encrypt_parts( + recipient_pub, + secret, + AES256_KEY_LEN, + true, + IV_LEN, + false, + 0, + 0, + )?; + let mut blob = KVec::new(); + blob.extend_from_slice(&parts.eph_point, GFP_KERNEL)?; + blob.extend_from_slice(&parts.ciphertext, GFP_KERNEL)?; + blob.extend_from_slice(&parts.tag, GFP_KERNEL)?; + Ok(blob) +} diff --git a/drivers/soc/apple/rxring.rs b/drivers/soc/apple/rxring.rs new file mode 100644 index 00000000000000..4aee7f407fa76f --- /dev/null +++ b/drivers/soc/apple/rxring.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use core::cell::UnsafeCell; +use kernel::soc::apple::mailbox::Message; +use kernel::sync::atomic::{Acquire, Atomic, Relaxed, Release}; + +const RING_LEN: u32 = 64; +const RING_MASK: u32 = RING_LEN - 1; + +pub(crate) struct RxRing { + slots: [UnsafeCell; RING_LEN as usize], + head: Atomic, + tail: Atomic, + dropped: Atomic, +} + +// SAFETY: ownership of each slot passes between the two sides via the +// release/acquire pairing on `head`/`tail`, so no slot is touched by both at +// once; single-producer/consumer are enforced by rx_lock and workqueue non-reentrancy. +unsafe impl Sync for RxRing {} +// SAFETY: `Message` is a plain data struct with no thread affinity. +unsafe impl Send for RxRing {} + +impl RxRing { + pub(crate) fn new() -> Self { + RxRing { + slots: core::array::from_fn(|_| UnsafeCell::new(Message { msg0: 0, msg1: 0 })), + head: Atomic::new(0), + tail: Atomic::new(0), + dropped: Atomic::new(0), + } + } + + pub(crate) fn push(&self, msg: Message) -> bool { + let head = self.head.load(Relaxed); + // Acquire pairs with the consumer's release of `tail`, so the reused slot is free. + let tail = self.tail.load(Acquire); + + if head.wrapping_sub(tail) >= RING_LEN { + self.dropped + .store(self.dropped.load(Relaxed).wrapping_add(1), Relaxed); + return false; + } + + let slot = &self.slots[(head & RING_MASK) as usize]; + // SAFETY: `head` is not published yet, so the consumer cannot be looking at + // this slot, and we are the only producer. + unsafe { slot.get().write(msg) }; + + // Release pairs with the consumer's acquire of `head`, publishing the slot write. + self.head.store(head.wrapping_add(1), Release); + true + } + + pub(crate) fn pop(&self) -> Option { + let tail = self.tail.load(Relaxed); + // Acquire pairs with the producer's release of `head`. + let head = self.head.load(Acquire); + + if head == tail { + return None; + } + + let slot = &self.slots[(tail & RING_MASK) as usize]; + // SAFETY: `head` is past `tail`, so the producer finished this slot and + // will not touch it again until we publish `tail`. + let msg = unsafe { slot.get().read() }; + + // Release pairs with the producer's acquire of `tail`, handing the slot back. + self.tail.store(tail.wrapping_add(1), Release); + Some(msg) + } + + pub(crate) fn dropped(&self) -> u32 { + self.dropped.load(Relaxed) + } +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs new file mode 100644 index 00000000000000..8b885ed98753e2 --- /dev/null +++ b/drivers/soc/apple/sbio.rs @@ -0,0 +1,3322 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. +//! The enclave matches; no biometric image ever crosses to userspace. + +use super::*; +use crate::proto::*; +use crate::sks::SKS_AUTH_TOKEN_LEN; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +impl SepData { + fn sbio_expect_ok(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_call(op) { + SbioOutcome::Ok(payload) => Some(payload), + _ => None, + } + } + + pub(crate) fn sbio_call(&self, op: &crate::sbio::SbioOp) -> SbioOutcome { + let done = match self.sbio_transfer(op) { + Ok(done) => done, + Err(_) => { + return SbioOutcome::Other; + } + }; + + let Some(err) = done.status.answered() else { + return SbioOutcome::Other; + }; + + match err as u16 { + crate::sbio::SBIO_STATUS_OK => SbioOutcome::Ok(done.payload), + crate::sbio::SBIO_STATUS_PREREQUISITE => SbioOutcome::PrerequisiteMissing, + crate::sbio::SBIO_STATUS_16 => SbioOutcome::Status16, + _ => SbioOutcome::Other, + } + } + + fn sbio_relay(&self, relay: &crate::sbio::SbioRelay<'_>) -> Option> { + match self.sbio_transfer_raw(relay.opcode(), relay.name(), relay.payload()) { + Ok(done) if done.status.is_ok() => Some(done.payload), + Ok(_) => None, + Err(_) => None, + } + } + + fn sync_device_view(&self) -> bool { + let synced = matches!( + self.sbio_call(&crate::sbio::sbio_update_device_list()), + SbioOutcome::Ok(_) + ); + + let policy_ok = match self.sbio_call(&crate::sbio::sbio_match_policy()) { + SbioOutcome::Ok(policy) if policy.len() == crate::sbio::SBIO_MATCH_POLICY_LEN => true, + SbioOutcome::Ok(_) => false, + _ => false, + }; + + synced && policy_ok + } + + fn note_capture_end(&self) { + self.last_capture_end_ns.store(shim::boottime_ns(), Relaxed); + } + + fn settle_before_capture(&self) { + let last = self.last_capture_end_ns.load(Relaxed); + if last == 0 { + return; + } + let elapsed_ms = shim::boottime_ns().saturating_sub(last) / 1_000_000; + if elapsed_ms >= u64::from(MATCH_SETTLE_MS) { + return; + } + let remaining = MATCH_SETTLE_MS - elapsed_ms as u32; + let _ = sensor::idle(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(remaining))); + } + + fn pace_between_captures(&self) { + let _ = sensor::idle(); + + if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, between captures") { + dev_warn!( + self.dev, + "enrol: sensor did not idle within {} ms; continuing the reposition wait anyway (pause is for the person)\n", + ENROL_IDLE_TIMEOUT_MS + ); + } + + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from( + ENROL_REPOSITION_MS, + ))); + } + + fn stash_enrol_identity(&self, record: &[u8]) { + let mut found: KVec<[u8; bio::UUID_LEN]> = KVec::new(); + crate::sbio::enrol_identity_candidates(record, SBIO_PROBE_USER_ID, |_at, uuid| { + let _ = found.push(uuid, GFP_KERNEL); + }); + *self.enrol_identity_candidates.lock() = found; + } + + fn identity_just_enrolled(&self) -> Option<[u8; bio::UUID_LEN]> { + let candidates = core::mem::take(&mut *self.enrol_identity_candidates.lock()); + let listed = self.enclave_identities_for(SBIO_PROBE_USER_ID)?; + + let mut confirmed: KVec<[u8; bio::UUID_LEN]> = KVec::new(); + for uuid in candidates.iter() { + if listed.iter().any(|u| u == uuid) && !confirmed.iter().any(|u| u == uuid) { + if confirmed.push(*uuid, GFP_KERNEL).is_err() { + break; + } + } + } + + if confirmed.len() == 1 { + return Some(confirmed[0]); + } + + let index = self.bio_index.lock(); + let mut new_identity = None; + for uuid in listed.iter() { + if index.contains_uuid(uuid) { + continue; + } + if new_identity.is_some() { + return None; + } + new_identity = Some(*uuid); + } + new_identity + } + + fn reconcile_identities(&self) { + let Some(listed) = self.enclave_identities_for(SBIO_PROBE_USER_ID) else { + return; + }; + + if listed.is_empty() { + return; + } + + let dropped = { + let mut index = self.bio_index.lock(); + match index.reconcile_to(&listed) { + Ok(dropped) => dropped, + Err(_) => { + return; + } + } + }; + + let held = { + let index = self.bio_index.lock(); + index.total() + }; + if dropped.is_empty() && held == listed.len() { + return; + } + let persisted = { + let index = self.bio_index.lock(); + self.with_host_store(|store| index.persist(store)) + }; + match persisted { + Some(Ok(())) => {}, + _ => dev_warn!( + self.dev, + "sbio: the reconciled index could not be written; it is correct in memory for this boot and will be rebuilt at the next attach\n" + ), + } + } + + fn enclave_identities_for(&self, user_id: i32) -> Option> { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + + let records = crate::sbio::IdentityRecords::new(&reply)?; + + let mut out = KVec::new(); + let mut failed = false; + records.each_built_in_for(user_id, |identity| { + if out.push(*identity.uuid(), GFP_KERNEL).is_err() { + failed = true; + } + }); + if failed { + return None; + } + Some(out) + } + + fn finish_enrolment(&self, outcome: core::result::Result<[u8; bio::UUID_LEN], u32>) { + let filed = { + let mut session = self.bio_session.lock(); + let mut index = self.bio_index.lock(); + bio::enrol_finish(&mut session, &mut index, outcome) + }; + if !filed { + return; + } + // lock ordering: store lock outside the session lock (UAF otherwise) + if outcome.is_ok() { + let index = self.bio_index.lock(); + let saved = self.with_host_store(|store| index.persist(store)); + match saved { + Some(Ok(())) => {}, + Some(Err(e)) => dev_err!( + self.dev, + "enrol: enclave holds a new template but the host index could not be written ({:?})\n", + e + ), + None => {} + } + } + self.bio_wake(); + } + + fn bio_wake(&self) { + if let Some(dev) = self.bio_dev.lock().as_ref() { + dev.wake(); + } + } + + pub(crate) fn attach_sensor(&self) { + if let Err(e) = sensor::register_driver() { + self.sensor_present.store(false, Relaxed); + dev_err!( + self.dev, + "sensor: could not register the SPI driver: {:?}\n", + e + ); + return; + } + + if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { + self.sensor_present.store(false, Relaxed); + dev_warn!( + self.dev, + "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", + sensor::CONTROLLER_BASE, + e + ); + return; + } + + let bound = sensor::is_bound(); + self.sensor_present.store(bound, Relaxed); + if bound && sensor::power_line().is_none() { + dev_warn!( + self.dev, + "sensor: no power line ({}); an unpowered sensor answers sixteen zero bytes like a dead bus\n", + sensor::power_source().name() + ); + } + } + + fn open_enrolment_context(&self, user: crate::sbio::UserId) -> bool { + let listed = match self.enclave_identities_for(SBIO_PROBE_USER_ID) { + Some(list) => list.len(), + None => { + return self.enrol_into_existing_context(user); + } + }; + + let Some(proof) = crate::sbio::NoExistingCatacomb::from_zero_identities(listed) else { + return self.enrol_into_existing_context(user); + }; + + self.open_fresh_context(user, &proof) + } + + fn enrol_into_existing_context(&self, user: crate::sbio::UserId) -> bool { + if !self.activate_protected_config(user) { + return false; + } + true + } + + fn open_fresh_context( + &self, + user: crate::sbio::UserId, + proof: &crate::sbio::NoExistingCatacomb, + ) -> bool { + let system = crate::sbio::sbio_select_context(crate::sbio::ContextScope::SYSTEM, proof); + if self.sbio_expect_ok(&system).is_none() { + return false; + } + + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb", + ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb", + ), + ] { + if !self.save_catacomb(who, kind, what) { + return false; + } + } + + let per_user = + crate::sbio::sbio_select_context(crate::sbio::ContextScope::user(user), proof); + if self.sbio_expect_ok(&per_user).is_none() { + return false; + } + + if !self.activate_protected_config(user) { + return false; + } + + true + } + + fn activate_protected_config(&self, user: crate::sbio::UserId) -> bool { + let op = crate::sbio::sbio_protected_config(user); + let Some(config) = self.sbio_expect_ok(&op) else { + return false; + }; + if config.len() != crate::sbio::SBIO_PROTECTED_CONFIG_LEN { + return false; + } + true + } + + fn begin_enrolment_on_enclave(&self) -> Option> { + if self.enrol_open.load(Relaxed) { + self.enrol_open.store(false, Relaxed); + let _ = self.sbio_call(&crate::sbio::sbio_cancel_operation()); + } + + let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID)?; + + // 0x03 answers -3 until a user key bag is designated + if !self.keybag_designated.load(Relaxed) { + return None; + } + + if self.enrol_material.lock().is_none() { + return None; + } + + // 0x03 answers 0x1 until an enrolment context exists + if !self.open_enrolment_context(user) { + return None; + } + + let Some(acm_handle) = self.establish_passcode_validated_context(user) else { + dev_err!( + self.dev, + "enrol: could not establish the PasscodeValidated SCRD credential; not falling back to a transient SKS token (a catacomb sealed under one does not survive a reboot)\n" + ); + return None; + }; + let op = + crate::sbio::sbio_begin_enrol(user, crate::sbio::BE_AUTH_TYPE_ACM_CONTEXT, &acm_handle); + + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.enrol_open.store(true, Relaxed); + Some(OpenEnrolment { + sep: self, + armed: true, + }) + } + _ => None, + } + } + + fn restore_all_components(&self) -> bool { + // 0x6b is re-read before every component, not once up front. + let user_id = SBIO_PROBE_USER_ID; + let mut any = false; + let mut missing_files = false; + let mut user_outcome = RestoreOutcome::NoStoredFile; + + // 0x8002 (cold transition) is a success only on the owner component. + for (id, kind, what, cold_ok) in [ + ( + crate::sbio::CatacombUser::MASTER.value(), + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb", + false, + ), + ( + crate::sbio::CatacombUser::OWNER.value(), + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb", + true, + ), + (user_id, PRIVATE_TYPE_CATACOMB_USER, c"user catacomb", false), + ] { + let Some(reply) = self.read_component_states() else { + return false; + }; + let states = crate::sbio::ComponentStates::new(&reply); + let outcome = self.restore_catacomb(states.state_for(id), id, kind, what, cold_ok); + match outcome { + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => any = true, + RestoreOutcome::NoStoredFile => missing_files = true, + RestoreOutcome::Failed | RestoreOutcome::EmptyTolerated => {} + } + if id == user_id { + user_outcome = outcome; + } + } + + let user_ok = matches!( + user_outcome, + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive + ); + let lockout = match self.restore_lockout() { + RestoreOutcome::Restored | RestoreOutcome::AlreadyActive => true, + RestoreOutcome::EmptyTolerated => true, + RestoreOutcome::NoStoredFile => { + missing_files = true; + false + } + RestoreOutcome::Failed => false, + }; + + if missing_files { + dev_warn!( + self.dev, + "sbio: at least one of four artefacts has no file on disk; if enrolled before the four-artefact save, re-enrol once to write all four\n" + ); + } + + let _ = any; + user_ok && lockout + } + + fn read_component_states(&self) -> Option> { + let op = crate::sbio::sbio_context_state(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + + let states = crate::sbio::ComponentStates::new(&reply); + if states.count() == 0 { + return None; + } + Some(reply) + } + + fn restore_catacomb( + &self, + state: Option, + id: i32, + kind: u8, + what: &CStr, + cold_ok: bool, + ) -> RestoreOutcome { + let Some(state) = state else { + return RestoreOutcome::Failed; + }; + match crate::sbio::component_action(state) { + crate::sbio::ComponentAction::AlreadyActive => RestoreOutcome::AlreadyActive, + crate::sbio::ComponentAction::Unsupported => RestoreOutcome::Failed, + crate::sbio::ComponentAction::Load => { + let Some(blob) = self.read_stored(kind, what) else { + return RestoreOutcome::NoStoredFile; + }; + + let at = crate::sbio::SBIO_SAVED_USER_ID_AT; + if blob.len() < at + 4 { + return RestoreOutcome::Failed; + } + let carried = + i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); + if carried != id { + return RestoreOutcome::Failed; + } + + let Some(request) = crate::sbio::SbioLoadCatacomb::new(&blob) else { + return RestoreOutcome::Failed; + }; + + match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { + Ok(done) if done.status.is_ok() => { + self.confirm_active(id, what, LoadAnswer::StatusZero) + } + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + && cold_ok => + { + self.confirm_active(id, what, LoadAnswer::ColdTransition) + } + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_COLD_TRANSITION) => + { + RestoreOutcome::Failed + } + // 0x101 ALREADY_ACTIVE is a success + Ok(done) + if done.status.answered() + == Some(crate::sbio::SBIO_STATUS_ALREADY_ACTIVE) => + { + self.confirm_active(id, what, LoadAnswer::AlreadyActive) + } + _ => RestoreOutcome::Failed, + } + } + } + } + + fn confirm_active(&self, id: i32, what: &CStr, answer: LoadAnswer) -> RestoreOutcome { + let Some(reply) = self.read_component_states() else { + dev_err!( + self.dev, + "sbio: the {} answered {} and 0x6b could not be re-read; treating unknown as failure\n", + what, + answer.describe() + ); + return RestoreOutcome::Failed; + }; + let states = crate::sbio::ComponentStates::new(&reply); + + match states.state_for(id) { + Some(state) if state & crate::sbio::COMPONENT_STATE_ACTIVE != 0 => { + RestoreOutcome::Restored + } + Some(state) if state & crate::sbio::COMPONENT_STATE_COLD != 0 => RestoreOutcome::Failed, + Some(_) => RestoreOutcome::Failed, + None => RestoreOutcome::Failed, + } + } + + fn restore_lockout(&self) -> RestoreOutcome { + let Some(blob) = self.read_stored(PRIVATE_TYPE_LOCKOUT, c"bio lockout record") else { + return RestoreOutcome::NoStoredFile; + }; + let Some(request) = crate::sbio::SbioLoadLockout::new(&blob) else { + return RestoreOutcome::Failed; + }; + match self.sbio_transfer_raw(request.opcode(), request.name(), request.payload()) { + Ok(done) if done.status.is_ok() => RestoreOutcome::Restored, + Ok(done) + if matches!( + done.status, + transfer::DeviceStatus::Answered(crate::sbio::SBIO_STATUS_COLD_TRANSITION) + ) => + { + RestoreOutcome::EmptyTolerated + } + Ok(_) => RestoreOutcome::Failed, + Err(_) => RestoreOutcome::Failed, + } + } + + fn read_stored(&self, kind: u8, _what: &CStr) -> Option> { + if crate::catacomb::is_kind(kind) { + if let Some(blob) = crate::catacomb::read(kind) { + return Some(blob); + } + } + match self.with_host_store(|store| store.read(&store::Key::root(kind))) { + Some(Ok(Some(blob))) => Some(blob), + Some(Ok(None)) => None, + Some(Err(_)) => None, + None => None, + } + } + + fn save_all_components(&self, user: crate::sbio::UserId) -> bool { + let mut all = true; + for (who, kind, what) in [ + ( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"master catacomb (user -1)", + ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner catacomb (user 501)", + ), + ( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"user catacomb", + ), + ] { + if !self.save_catacomb(who, kind, what) { + all = false; + } + } + if !self.save_lockout() { + all = false; + } + all + } + + fn save_after_match(&self, user: crate::sbio::UserId) -> bool { + if !self.save_lockout() { + dev_err!( + self.dev, + "verify: post-match lockout persistence failed; refusing to claim the AP and SEP anti-replay state are synchronized\n" + ); + return false; + } + + if !self.save_catacomb( + crate::sbio::CatacombUser::enrolling(user), + PRIVATE_TYPE_CATACOMB_USER, + c"post-match user catacomb", + ) { + dev_err!( + self.dev, + "verify: post-match user-catacomb persistence failed; the next boot may reject the older AP blob\n" + ); + return false; + } + + if !self.save_catacomb( + crate::sbio::CatacombUser::MASTER, + PRIVATE_TYPE_CATACOMB_MASTER, + c"post-match system (id -1) material snapshot", + ) { + dev_err!( + self.dev, + "verify: post-match system material snapshot failed; next boot may reinstall pre-match material and the user load could cold-transition (0x8002)\n" + ); + return false; + } + + if !self.resnapshot_identity_keybag() { + dev_err!( + self.dev, + "verify: post-match identity-bag snapshot failed; the newly confirmed catacomb has no matching durable bag image\n" + ); + return false; + } + + true + } + + fn save_catacomb(&self, who: crate::sbio::CatacombUser, kind: u8, _what: &CStr) -> bool { + let selector = crate::sbio::SaveSelector::new(who); + + let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_catacomb(&selector)) else { + return false; + }; + if blob.len() < crate::sbio::SBIO_SAVED_MIN || blob.len() > crate::sbio::SBIO_SAVED_MAX { + return false; + } + + let at = crate::sbio::SBIO_SAVED_USER_ID_AT; + let carried = i32::from_le_bytes([blob[at], blob[at + 1], blob[at + 2], blob[at + 3]]); + if carried != who.value() { + return false; + } + + if crate::catacomb::write(kind, &blob).is_err() { + return false; + } + + if self + .sbio_expect_ok(&crate::sbio::sbio_confirm_save(&selector)) + .is_none() + { + return false; + } + true + } + + fn save_lockout(&self) -> bool { + let Some(blob) = self.sbio_expect_ok(&crate::sbio::sbio_save_lockout()) else { + return false; + }; + if blob.is_empty() { + return false; + } + match self + .with_host_store(|store| store.write(&store::Key::root(PRIVATE_TYPE_LOCKOUT), &blob)) + { + Some(Ok(())) => true, + Some(Err(_)) => false, + None => false, + } + } + + fn enable_sbio(&self) -> Result<()> { + if self.sbio_ready.load(Relaxed) { + return Ok(()); + } + + self.register_ool(&self.ool_sbio)?; + + self.sbio_ready.store(true, Relaxed); + Ok(()) + } + + fn attach_bringup(&self) { + if !self.sensor_present.load(Relaxed) { + dev_warn!( + self.dev, + "sbio: no sensor bound at attach; catacomb restore ran but capture is unavailable\n" + ); + return; + } + let Some(patch) = self.wake_sensor() else { + return; + }; + if !self.complete_bringup(patch) { + dev_warn!( + self.dev, + "sbio: attach-time sensor bring-up did not complete; matching remains unavailable until a later bring-up succeeds.\n" + ); + } + } + + fn bring_sensor_online(&self) -> bool { + let Some(mut patch) = self.wake_sensor() else { + return false; + }; + + if !self.sensor_calibrated.load(Relaxed) { + if !self.calibrate_sensor() { + return false; + } + self.sensor_calibrated.store(true, Relaxed); + let Some(reloaded_patch) = self.wake_sensor() else { + return false; + }; + patch = reloaded_patch; + } + + let ok = self.complete_bringup(patch); + // The sensor is patched and idle here -- the only safe moment to + // configure the data-ready interrupt, which is then left alone for the + // driver's life. Interrupt capture is the default; a machine that does + // not describe a data-ready line falls back to SPI polling. + if ok && !sensor::irq_available() { + if sensor::irq_setup() { + dev_info!(self.dev, "sensor: interrupt-driven capture\n"); + } else { + dev_warn!( + self.dev, + "sensor: no data-ready interrupt (describe it in DT); capture falls back to SPI polling\n" + ); + } + } + ok + } + + pub(crate) fn run_bringup(&self) { + if self.bringup_started.xchg(true, Relaxed) { + return; + } + if let Err(e) = self.enable_sks() { + dev_warn!(self.dev, "bringup: could not enable the key store ({:?}); keybag and ref-key operations are unavailable\n", e); + return; + } + if *module_parameters::provision_keybag.value() != 0 { + if *module_parameters::xart_writes.value() == 0 { + dev_err!( + self.dev, + "bringup: provision_keybag=1 requires xart_writes=1\n" + ); + return; + } + if !self.sks_provision_identity_keybag() { + dev_err!( + self.dev, + "bringup: identity-keybag provisioning failed; no retry this boot\n" + ); + return; + } + } + if let Err(e) = self.register_bio() { + dev_err!( + self.dev, + "bringup: could not publish /dev/{} after SKS became ready: {:?}\n", + bio::DEVICE_NAME, + e + ); + } + } + + fn activate_touchid(&self) -> Result<()> { + if self.touchid_started.load(Relaxed) { + return Ok(()); + } + if self.touchid_failed.load(Relaxed) { + return Err(EIO); + } + + let mut store = store::Store::open().inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: opening the host state failed: {:?}\n", + e + ); + })?; + let index = bio::IdentityIndex::load(&mut store).inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: loading the host identity index failed: {:?}\n", + e + ); + })?; + *self.host_store.lock() = Some(store); + *self.bio_index.lock() = index; + + self.attach_sensor(); + self.enable_sbio().inspect_err(|e| { + dev_err!( + self.dev, + "Touch ID: registering the biometric buffers failed: {:?}\n", + e + ); + })?; + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(stored)) => stored, + Ok(keybag::State::Absent(_)) => { + dev_err!(self.dev, "Touch ID: no persisted identity keybag exists\n"); + return Err(ENOENT); + } + Err(e) => { + dev_err!( + self.dev, + "Touch ID: reading the persisted identity keybag failed: {:?}\n", + e + ); + return Err(e); + } + }; + if !self.sks_ready() { + dev_err!( + self.dev, + "Touch ID: the key-store endpoint is unavailable\n" + ); + return Err(ENODEV); + } + let (handle, uuid) = self.sks_recover(&stored).ok_or_else(|| { + dev_err!( + self.dev, + "Touch ID: the persisted identity keybag did not recover\n" + ); + EIO + })?; + self.sks_designate_user_keybag(handle, stored.secret()); + self.sks_machine_refkey(handle, stored.secret()); + let prepared = self.cold_match_continue(handle, uuid); + self.ensure_restored_after(prepared); + self.attach_bringup(); + self.touchid_started.store(true, Relaxed); + Ok(()) + } + + fn prepare_bio_open(&self) -> Result<()> { + if let Err(e) = self.activate_touchid() { + self.touchid_failed.store(true, Relaxed); + dev_err!( + self.dev, + "Touch ID activation failed: {:?}; refusing retries this boot because a partial keybag load is not safely repeatable\n", + e + ); + return Err(e); + } + Ok(()) + } + + pub(crate) fn run_verify(&self) { + if !self.templates_restored.load(Relaxed) { + dev_err!( + self.dev, + "verify: refusing before touching the sensor; cold-match prep or restore did not complete, enclave holds no template (every match refused 0x1). A restore failure, not a non-matching finger\n" + ); + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + [0u8; bio::TOKEN_LEN], + ); + return; + } + + let Some(token_bytes) = self.mint_token_bytes() else { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + [0u8; bio::TOKEN_LEN], + ); + return; + }; + + self.settle_before_capture(); + + // The enclave refuses a capture from an uncalibrated sensor (0x65 answers 1). + if !self.bring_sensor_online() { + self.finish_verify(bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), token_bytes); + let _ = sensor::idle(); + return; + } + + // The match uses the plain 0x23->0x06->0x65 pipeline, with no SCRD step. + + let outcome = self.verify_one_image(); + let _ = sensor::idle(); + sensor::power(false); + self.note_capture_end(); + + let definitive = matches!( + &outcome, + bio::VerifyOutcome::Matched(_) | bio::VerifyOutcome::NoMatch + ); + if definitive { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + token_bytes, + ); + return; + }; + if !self.save_after_match(user) { + self.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE), + token_bytes, + ); + return; + } + } + self.finish_verify(outcome, token_bytes); + } + + fn verify_one_image(&self) -> bio::VerifyOutcome { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if sensor::start_capture().is_err() { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + let advertised = match self.await_capture() { + CaptureWait::Ready(n) => n, + CaptureWait::Timeout => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_TIMEOUT); + } + CaptureWait::Fault(_state) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), + }; + + let capture = match sensor::read_capture(advertised) { + Ok(capture) => capture, + Err(sensor::CaptureError::Checksum { .. }) | Err(sensor::CaptureError::Length(_)) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + Err(sensor::CaptureError::Bus(_)) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR); + } + Err(sensor::CaptureError::NoMemory) => { + return bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR) + } + }; + + if self + .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .is_none() + { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let _image = ImageContext { sep: self }; + + // MATCHING: byte 0x12 set, 0x13 clear; last-image flag at byte 9 (enrol byte 8) + let init = crate::sbio::sbio_image_processing_init( + crate::sbio::ImagePurpose::Matching, + true, + true, + 0, + user, + crate::shim::monotonic_ns(), + ); + if self.sbio_expect_ok(&init).is_none() { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + let relay = crate::sbio::SbioRelay::new(&capture); + if self.sbio_relay(&relay).is_none() { + dev_err!( + self.dev, + "verify: enclave refused the capture (status above); no comparison performed, not a non-match. Reported as a failure so userspace does not tell the user their finger was rejected\n" + ); + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + drop(capture); + + let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if assessment.len() <= crate::sbio::ASSESS_USABLE_MATCH { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + if assessment[crate::sbio::ASSESS_USABLE_MATCH] == 0 { + return bio::VerifyOutcome::Failed(ENROL_STATUS_RETRY); + } + + let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_match_result()) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + let Some(parsed) = crate::sbio::MatchResult::parse(&result) else { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if !parsed.matches(user) { + return bio::VerifyOutcome::NoMatch; + } + + // match result: user id groups templates, UUID at offset 0x04 identifies one + let identity = *parsed.identity_uuid(); + let known = self.bio_index.lock().contains_uuid(&identity); + + if !known { + return bio::VerifyOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + bio::VerifyOutcome::Matched(bio::MatchEvidence::from_enclave_reply(identity)) + } + + pub(crate) fn run_enrolment(&self) { + if !self.bring_sensor_online() { + self.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + let _ = sensor::idle(); + return; + } + + let Some(open) = self.begin_enrolment_on_enclave() else { + self.finish_enrolment(Err(ENROL_STATUS_ENCLAVE)); + let _ = sensor::idle(); + return; + }; + + let mut counter: u32 = 0; + let mut enrolment_completed = false; + let mut identity: Option<[u8; bio::UUID_LEN]> = None; + + let outcome = loop { + if !bio::enrol_is_live(&self.bio_session.lock()) { + break None; + } + if counter >= ENROL_MAX_CAPTURES { + break Some(Err(ENROL_STATUS_TOO_MANY)); + } + + match self.enrol_one_image(counter) { + ImageOutcome::Progress { + stage, + percent, + complete, + has_template, + } => { + counter = counter.saturating_add(1); + let woke = bio::enrol_advance( + &mut self.bio_session.lock(), + stage, + percent, + bio::Guidance::LiftAndMove, + ); + if woke { + self.bio_wake(); + } + // completion is the flag at offset 0xbfe, not a derived stage count + if complete { + if !has_template { + break Some(Err(ENROL_STATUS_ENCLAVE)); + } + enrolment_completed = true; + break Some(Ok(())); + } + } + ImageOutcome::Retry => { + counter = counter.saturating_add(1); + // The capture did not take (partial/unusable contact). Nudge + // the person to hold still and press again rather than + // silently re-capturing with no feedback. + if bio::enrol_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + self.bio_wake(); + } + } + ImageOutcome::NoFinger => { + break Some(Err(ENROL_STATUS_TIMEOUT)); + } + ImageOutcome::Failed(status) => break Some(Err(status)), + } + + self.pace_between_captures(); + }; + + if enrolment_completed { + open.completed(); + + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + if !self.save_all_components(user) { + dev_warn!( + self.dev, + "enrol: enrolment succeeded but at least one of four artefacts was not persisted; works until the next reboot (see which component above)\n" + ); + } else { + self.templates_restored.store(true, Relaxed); + if !self.resnapshot_identity_keybag() { + dev_warn!( + self.dev, + "enrol: catacombs saved but post-commit identity-bag snapshot was not; template works this boot but is not reboot-persistent\n" + ); + } + } + } + + identity = self.identity_just_enrolled(); + + self.reconcile_identities(); + } + + let _ = sensor::idle(); + sensor::power(false); + self.note_capture_end(); + + if let Some(result) = outcome { + let filed = match (result, identity) { + (Ok(()), Some(uuid)) => Ok(uuid), + (Ok(()), None) => Err(ENROL_STATUS_UNFILED), + (Err(status), _) => Err(status), + }; + self.finish_enrolment(filed); + } + } + + fn register_sensor(&self, id: &sensor::Identifier) -> bool { + let stage = self.bringup.load(Relaxed); + + if stage >= BRINGUP_ESTABLISHED { + if self + .sbio_expect_ok(&crate::sbio::sbio_clear_state()) + .is_none() + { + return false; + } + } + + if stage == BRINGUP_FRESH { + let op = crate::sbio::sbio_register_sensor(id); + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); + } + _ => { + return false; + } + } + } + + let serial = match sensor::read_sensor_serial() { + Ok(serial) => serial, + Err(_) => { + return false; + } + }; + + let op = crate::sbio::sbio_register_sensor_serial(&serial); + match self.sbio_call(&op) { + SbioOutcome::Ok(_payload) => { + self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + true + } + _ => false, + } + } + + fn wake_sensor(&self) -> Option { + let _source = sensor::power_source(); + let cycled = sensor::power_cycle(); + if !sensor::cs_timing().is_hardware() { + dev_warn!( + self.dev, + "sensor: chip-select timing not reaching hardware; software emulation cannot time this sensor, silent status expected (mode 2 is the other half)\n" + ); + } + + let mut answered_without_identifier = false; + + for delay in sensor::POWER_ON_READ_DELAYS_MS { + if delay > 0 { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(delay))); + } + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return None; + } + }; + if st.is_silent() { + continue; + } + + if let sensor::Offset12::Identifier(id) = st.offset12() { + if id != 0 && !sensor::KNOWN_IDENTIFIERS.contains(&id) { + return None; + } + } + + let Some(id) = st.identifier_to_register() else { + answered_without_identifier = true; + continue; + }; + if !self.register_sensor(&id) { + return None; + } + + // The patch decision is made on status[8], not on state 9. + if st.patch_ack() == sensor::PATCH_ACCEPTED { + return Some(PatchLoaded(())); + } + return self.apply_sensor_patch(st.patch_ack()); + } + + dev_err!( + self.dev, + "sensor gated: all {} wake reads returned sixteen zero bytes. Power {}. CS timing: {}. Mode 2 (CPOL=1 CPHA=0)\n", + sensor::POWER_ON_READ_DELAYS_MS.len(), + if cycled { c"was cycled" } else { c"was NOT cycled" }, + sensor::cs_timing().name() + ); + if answered_without_identifier { + return None; + } + None + } + + fn calibrate_sensor(&self) -> bool { + // The blob is per-device factory data, so its name is board data: take + // it from the sensor node's firmware-name, falling back to the default. + let name = sensor::firmware_name().unwrap_or(CALIBRATION_FIRMWARE); + let blob = match kernel::firmware::Firmware::request(name, &self.dev) { + Ok(fw) => CalibrationBlob::new(fw), + Err(e) => { + dev_err!( + self.dev, + "sensor: calibration blob {} could not be loaded ({:?}); per-device factory data, cannot be synthesised, stopping\n", + name, + e + ); + return false; + } + }; + + let Some(challenge) = self.sbio_expect_ok(&crate::sbio::sbio_module_challenge()) else { + return false; + }; + if challenge.len() != crate::sbio::SBIO_MODULE_CHALLENGE_LEN { + return false; + } + + let challenge = sensor::Params::new(challenge); + if sensor::send_module_challenge(&challenge).is_err() { + return false; + } + + let reply = match sensor::read_module_reply() { + Ok(reply) => reply, + Err(_) => { + return false; + } + }; + + // commit the sensor's reply, not the challenge; both are 64 bytes at count 0x4b + let Some(commit) = crate::sbio::sbio_module_commit(&reply) else { + return false; + }; + let Some(_serial) = self.sbio_expect_ok(&commit) else { + return false; + }; + + let request = match crate::sbio::SbioCalibration::new(&blob) { + Ok(request) => request, + Err(_) => { + return false; + } + }; + if self + .sbio_transfer_raw(request.opcode(), request.name(), request.payload()) + .ok() + .filter(|done| done.status.is_ok()) + .is_none() + { + dev_err!( + self.dev, + "sensor: 0x5b LOAD_CALIBRATION failed. Without it every capture is refused with status 1.\n" + ); + return false; + } + + let _ = self.sbio_call(&crate::sbio::sbio_diagnostics()); + + true + } + + fn complete_bringup(&self, patch: PatchLoaded) -> bool { + let Some(params) = self.apply_sensor_parameters() else { + return false; + }; + + let op = crate::sbio::sbio_complete_init(patch, params); + if self.sbio_expect_ok(&op).is_none() { + dev_err!( + self.dev, + "sensor: 0x01 COMPLETE_INIT failed — status above\n" + ); + return false; + } + + let listed = self.sync_device_view(); + + if listed && !self.device_view_synced.xchg(true, Relaxed) { + let count = self.log_identity_count(c"after device-view synchronisation"); + match count { + Some(0) => { + self.templates_restored.store(false, Relaxed); + } + Some(n) if self.templates_restored.load(Relaxed) && self.prove_restore() => { + dev_warn!( + self.dev, + "Touch ID: restored {} enrolled identity/identities; enrolment survived reboot\n", + n + ); + } + Some(_) => { + self.templates_restored.store(false, Relaxed); + } + None => { + self.templates_restored.store(false, Relaxed); + } + } + if self.templates_restored.load(Relaxed) { + self.reconcile_identities(); + } + } + + true + } + + fn apply_sensor_parameters(&self) -> Option { + // 0x5d and 0x5c: an empty blob is a failure. + self.relay_parameters( + &crate::sbio::sbio_coverage_params(), + &sensor::Geometry::COVERAGE, + false, + )?; + self.relay_parameters( + &crate::sbio::sbio_operation_params(), + &sensor::Geometry::OPERATION, + false, + )?; + // 0x6a: an empty blob is success, and relaying nothing is correct. + self.relay_parameters( + &crate::sbio::sbio_transparent_channel(), + &sensor::Geometry::TRANSPARENT, + true, + )?; + + Some(ParametersApplied(())) + } + + fn relay_parameters( + &self, + op: &crate::sbio::SbioOp, + geom: &sensor::Geometry, + empty_is_success: bool, + ) -> Option { + let blob = match self.sbio_call(op) { + SbioOutcome::Ok(b) => sensor::Params::new(b), + _ => { + return None; + } + }; + + if blob.is_empty() { + if empty_is_success { + return Some(0); + } + return None; + } + + let relayed = blob.len(); + match sensor::send_encrypted_parameters(&blob, geom) { + Ok(()) => Some(relayed), + Err(sensor::ParamsError::Empty) => None, + Err(sensor::ParamsError::TooLong(n, capacity)) => { + dev_err!( + self.dev, + "sensor: {}'s blob is {} bytes and this frame holds {} (declared 0x{:x} less the 9 bytes of header and CRC)\n", + op.name(), + n, + capacity, + geom.declared() + ); + None + } + Err(sensor::ParamsError::Transfer(_e)) => None, + } + } + + fn establish_session(&self) -> bool { + let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { + SbioOutcome::Ok(sh) => sh, + SbioOutcome::Status16 => { + return false; + } + SbioOutcome::PrerequisiteMissing => { + return false; + } + SbioOutcome::Other => { + return false; + } + }; + + // 0x30 reserved for this reply; length is a floor, take the first 40 + if share.len() < sensor::SESSION_SHARE_LEN { + return false; + } + let mut out = [0u8; sensor::SESSION_SHARE_LEN]; + out.copy_from_slice(&share[..sensor::SESSION_SHARE_LEN]); + + // relay it: class 0x72, no CRC, no padding + if sensor::send_session_share(&out).is_err() { + return false; + } + + let reply = match sensor::read_session_reply() { + Ok(r) => r, + Err(_) => { + return false; + } + }; + + match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { + Ok(done) if done.status.is_ok() => true, + Ok(_) => false, + Err(_) => false, + } + } + + fn init_sequence_counter(&self) -> bool { + let challenge = match self.sbio_call(&crate::sbio::sbio_request_challenge()) { + SbioOutcome::Ok(c) => c, + SbioOutcome::PrerequisiteMissing => { + if !self.establish_session() { + return false; + } + match self.sbio_call(&crate::sbio::sbio_request_challenge()) { + SbioOutcome::Ok(c) => c, + _ => { + return false; + } + } + } + SbioOutcome::Status16 => { + return false; + } + SbioOutcome::Other => { + return false; + } + }; + if challenge.len() < sensor::CHALLENGE_LEN { + return false; + } + let mut out = [0u8; sensor::CHALLENGE_LEN]; + out.copy_from_slice(&challenge[..sensor::CHALLENGE_LEN]); + + if sensor::send_challenge(&out).is_err() { + return false; + } + + let reply = match sensor::read_challenge_reply() { + Ok(r) => r, + Err(_) => { + return false; + } + }; + + match self.sbio_transfer(&crate::sbio::sbio_commit_challenge(&reply)) { + Ok(done) if done.status.is_ok() => true, + Ok(_) => false, + Err(_) => false, + } + } + + fn apply_sensor_patch(&self, _ack_before: u8) -> Option { + let blob = match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { + SbioOutcome::Ok(b) => b, + SbioOutcome::PrerequisiteMissing => { + if !self.init_sequence_counter() { + return None; + } + match self.sbio_call(&crate::sbio::sbio_fetch_patch()) { + SbioOutcome::Ok(b) => b, + _ => { + return None; + } + } + } + SbioOutcome::Status16 => { + return None; + } + SbioOutcome::Other => { + return None; + } + }; + if blob.is_empty() { + return None; + } + + // enable command is mandatory; skipping it leaves the sensor in state 9 + if sensor::setup_patch_enable().is_err() { + return None; + } + + if !self.await_sensor_state(sensor::STATE_IDLE, c"idle, before sending the patch") { + return None; + } + + if sensor::send_patch(&blob).is_err() { + return None; + } + + // acceptance is at status[8], not at the state + for _attempt in 0..PATCH_POLL_ATTEMPTS { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(PATCH_POLL_MS))); + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED { + if let Ok(after) = sensor::status() { + if after.state == sensor::STATE_NEEDS_PATCH { + return None; + } + } + return Some(PatchLoaded(())); + } + } + + let _ = sensor::status(); + None + } + + fn await_sensor_state(&self, want: u8, _why: &CStr) -> bool { + for _attempt in 0..PATCH_POLL_ATTEMPTS { + match sensor::status() { + Ok(st) if st.state == want => { + return true; + } + Ok(_) => {} + Err(_) => { + return false; + } + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(PATCH_POLL_MS))); + } + false + } + + fn enrol_one_image(&self, counter: u32) -> ImageOutcome { + if sensor::start_capture().is_err() { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + + let available = match self.await_capture() { + CaptureWait::Ready(n) => n, + CaptureWait::Timeout => return ImageOutcome::NoFinger, + CaptureWait::Fault(_) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + CaptureWait::Abandon => return ImageOutcome::Failed(ENROL_STATUS_SENSOR), + }; + + let capture = match sensor::read_capture(available) { + Ok(c) => c, + Err(sensor::CaptureError::Checksum { + advertised: _advertised, + computed: _computed, + }) => { + return ImageOutcome::Retry; + } + Err(sensor::CaptureError::Length(_n)) => { + return ImageOutcome::Retry; + } + Err(sensor::CaptureError::Bus(_e)) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR); + } + Err(sensor::CaptureError::NoMemory) => { + return ImageOutcome::Failed(ENROL_STATUS_SENSOR) + } + }; + + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if self + .sbio_expect_ok(&crate::sbio::sbio_prepare_image_processing()) + .is_none() + { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let _image = ImageContext { sep: self }; + + let init = crate::sbio::sbio_image_processing_init( + crate::sbio::ImagePurpose::Enrolment, + counter == 0, + false, + counter, + user, + crate::shim::monotonic_ns(), + ); + if self.sbio_expect_ok(&init).is_none() { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + + let relay = crate::sbio::SbioRelay::new(&capture); + if self.sbio_relay(&relay).is_none() { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + drop(capture); + + let Some(assessment) = self.sbio_expect_ok(&crate::sbio::sbio_image_assessment()) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + if assessment.len() < crate::sbio::ASSESS_MIN_LEN { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + } + let usable = assessment[crate::sbio::ASSESS_USABLE_ENROL] != 0; + if !usable { + return ImageOutcome::Retry; + } + + let Some(result) = self.sbio_expect_ok(&crate::sbio::sbio_enrolment_result()) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + let Some(parsed) = crate::sbio::EnrolmentResult::parse(&result) else { + return ImageOutcome::Failed(ENROL_STATUS_ENCLAVE); + }; + + if parsed.complete() { + self.stash_enrol_identity(&result); + } + + let percent = parsed.progress_percent(); + + ImageOutcome::Progress { + stage: (percent * bio::ENROL_STAGES) + .div_ceil(100) + .min(bio::ENROL_STAGES), + percent, + complete: parsed.complete(), + has_template: parsed.has_template(), + } + } + + fn await_capture(&self) -> CaptureWait { + let mut previous: Option<[u8; sensor::STATUS_LEN]> = None; + let mut armed_reported = false; + + // Interrupt-driven capture blocks on the data-ready line instead of + // polling; the SPI status read below still gates every frame, so a + // missed or spurious interrupt costs at most one backstop interval. A + // machine with no data-ready line falls back to SPI polling. + let use_irq = sensor::irq_available(); + if use_irq { + sensor::irq_arm(); + } + + // Bound the wait by wall-clock time, not a fixed iteration count. The + // data-ready interrupt can wake early on a stray edge (the sensor + // toggles the line as the finger moves between enrol frames); a fixed + // attempt budget would then be spent in milliseconds and time the + // capture out before the frame lands. Time-bounding degrades, at worst, + // to the same duration as the poll path. + let deadline_ns = crate::shim::monotonic_ns().saturating_add( + u64::from(ENROL_POLL_ATTEMPTS) * u64::from(ENROL_POLL_MS) * 1_000_000, + ); + let mut attempt: u32 = 0; + loop { + if !bio::capture_is_live(&self.bio_session.lock()) { + return CaptureWait::Abandon; + } + + let st = match sensor::status() { + Ok(st) => st, + Err(_) => { + return CaptureWait::Abandon; + } + }; + + if attempt == 0 || previous != Some(st.raw) { + previous = Some(st.raw); + } + + if st.state == sensor::STATE_NEEDS_PATCH { + return CaptureWait::Fault(st.state); + } + + let guide = match st.state { + sensor::STATE_ARMED => Some(bio::Guidance::Place), + sensor::STATE_READING => Some(bio::Guidance::HoldStill), + _ => None, + }; + if let Some(guide) = guide { + if bio::enrol_guide(&mut self.bio_session.lock(), guide) { + self.bio_wake(); + } + } + + if st.state == sensor::STATE_ARMED && !armed_reported { + armed_reported = true; + } + + if let sensor::Offset12::Available(count) = st.offset12() { + if count == 0 { + return CaptureWait::Timeout; + } + return CaptureWait::Ready(count); + } + + if crate::shim::monotonic_ns() >= deadline_ns { + break; + } + + if use_irq { + // Block on the data-ready line; the interrupt wakes this the + // instant a frame is ready, and the backstop bounds a missed + // edge. Re-arm for the next frame. + let _ = sensor::irq_wait(ENROL_IRQ_WAIT_MS); + sensor::irq_arm(); + } else { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(ENROL_POLL_MS))); + } + attempt = attempt.saturating_add(1); + } + let _ = sensor::status(); + CaptureWait::Timeout + } + + pub(crate) fn on_sbio(&self, msg: Message) { + let f = proto::decode(&msg); + let marker = f.tag; + + if !self.sbio_ready.load(Relaxed) { + return; + } + + // notification and the enclave's DMA are not ordered; buffer may still be poison + if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { + self.sbio_fail_unwritten(); + return; + } + + let header = match self.ool_read(&self.ool_sbio, 0, transfer::HEADER_LEN) { + Ok(h) => h, + Err(_) => { + self.sbio_fail(); + return; + } + }; + let packet = match transfer::Packet::decode(&header) { + Ok(p) => p, + Err(_) => { + self.sbio_fail(); + return; + } + }; + + let chunk = packet.chunk as usize; + let payload = if chunk == 0 { + KVec::new() + } else if !self.ool_await_written(&self.ool_sbio, transfer::HEADER_LEN, chunk) { + self.sbio_fail_unwritten(); + return; + } else { + match self.ool_read(&self.ool_sbio, transfer::HEADER_LEN, chunk) { + Ok(p) => p, + Err(_) => { + self.sbio_fail(); + return; + } + } + }; + + let progress = self.sbio_rx.lock().on_chunk(marker, &packet, &payload); + + match progress { + transfer::Progress::NeedMore(cont) => { + self.sbio_continue(&cont); + } + transfer::Progress::Complete => { + // 0xFE requests the peer's next packet and acks the final one; nothing to send + self.sbio_wq.notify_all(); + } + transfer::Progress::Ignored => {} + transfer::Progress::Grant => { + self.sbio_wq.notify_all(); + } + transfer::Progress::Notification => {} + transfer::Progress::Failed => { + self.sbio_wq.notify_all(); + } + } + } + + fn sbio_continue(&self, cont: &transfer::Continuation) { + let mut header = [0u8; transfer::HEADER_LEN]; + if cont.packet().encode(&mut header).is_err() { + self.sbio_fail(); + return; + } + if self.ool_write(&self.ool_sbio, 0, &header).is_err() { + self.sbio_fail(); + return; + } + if self.send(crate::sbio::encode_sbio_continue(cont)).is_err() { + self.sbio_fail(); + } + } + + fn sbio_fail(&self) { + self.sbio_rx.lock().abort(); + self.sbio_wq.notify_all(); + } + + fn sbio_fail_unwritten(&self) { + self.sbio_rx + .lock() + .abort_with(transfer::DeviceStatus::BufferNeverWritten); + self.sbio_wq.notify_all(); + } + + fn sbio_transfer(&self, op: &crate::sbio::SbioOp) -> Result { + self.sbio_transfer_raw(op.opcode(), op.name(), op.payload()) + } + + fn sbio_transfer_raw( + &self, + opcode: u16, + name: &'static CStr, + payload: &[u8], + ) -> Result { + if !self.sbio_ready.load(Relaxed) { + return Err(ENODEV); + } + + self.sbio_rx.lock().begin(u32::from(opcode))?; + + if let Err(e) = self.sbio_start(opcode, name, payload) { + let mut guard = self.sbio_rx.lock(); + guard.abort(); + let _ = guard.take_done(); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); + let mut guard = self.sbio_rx.lock(); + loop { + if let Some(done) = guard.take_done() { + return Ok(done); + } + if remaining == 0 { + guard.abort(); + let _ = guard.take_done(); + drop(guard); + return Err(ETIMEDOUT); + } + match self + .sbio_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn sbio_start(&self, opcode: u16, name: &'static CStr, payload: &[u8]) -> Result<()> { + if crate::sbio::encode_sbio_raw(opcode, 0).is_none() { + return Err(EPERM); + } + + let total = payload.len(); + if total > transfer::MAX_TRANSACTION as usize { + return Err(EINVAL); + } + + let capacity = OOL_SIZE_SBIO - transfer::HEADER_LEN; + + self.sbio_rx.lock().begin_send(u32::from(opcode)); + let result = self.sbio_send_chunks(opcode, name, payload, capacity); + self.sbio_rx.lock().end_send(); + result + } + + fn sbio_send_chunks( + &self, + opcode: u16, + name: &'static CStr, + payload: &[u8], + capacity: usize, + ) -> Result<()> { + let total = payload.len(); + let mut offset = 0usize; + let mut first = true; + let mut seq: u16 = 0; + let mut request = KVec::new(); + + loop { + let chunk = core::cmp::min(total - offset, capacity); + + let packet = transfer::Packet { + version: 1, + total: total as u32, + offset: offset as u32, + flags: 0, + err: 0, + opcode: u32::from(opcode), + chunk: chunk as u32, + }; + + request.clear(); + request.resize(transfer::HEADER_LEN + chunk, 0u8, GFP_KERNEL)?; + packet.encode(&mut request)?; + request[transfer::HEADER_LEN..].copy_from_slice(&payload[offset..offset + chunk]); + + self.ool_write(&self.ool_sbio, 0, &request)?; + + let msg = if first { + crate::sbio::encode_sbio_raw(opcode, seq) + } else { + crate::sbio::encode_sbio_next(opcode, seq) + } + .ok_or(EPERM)?; + self.send(msg)?; + + offset += chunk; + first = false; + seq = seq.wrapping_add(1); + if offset >= total { + return Ok(()); + } + + self.sbio_await_grant(name, offset, total)?; + } + } + + fn sbio_await_grant(&self, _name: &'static CStr, _sent: usize, _total: usize) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); + let mut guard = self.sbio_rx.lock(); + loop { + if guard.take_grant() { + return Ok(()); + } + if remaining == 0 { + drop(guard); + return Err(ETIMEDOUT); + } + match self + .sbio_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + pub(crate) fn register_bio(&self) -> Result<()> { + let mut guard = self.bio_dev.lock(); + if guard.is_some() { + return Ok(()); + } + + let ctx = core::ptr::from_ref(self).cast_mut().cast::(); + + // SAFETY: `ctx` is this `SepData`, kept alive by the `Arc` in the + // driver's private data. `remove()` drops the registration before that + // `Arc` can go, and the shim's unregister clears its stored context + // under the same lock every callback takes, so once it returns, no + // callback can observe this pointer again. + let dev = unsafe { + shim::BioChardev::register( + bio::DEVICE_NAME, + bio::DEVICE_MODE, + ctx, + bio_open_trampoline, + bio_release_trampoline, + bio_ioctl_trampoline, + bio_ready_trampoline, + ) + }?; + + *guard = Some(dev); + Ok(()) + } + + pub(crate) fn bio_open(&self) -> Result<()> { + let mut session = self.bio_session.lock(); + bio::open(&mut session)?; + drop(session); + if let Err(e) = self.prepare_bio_open() { + bio::release(&mut self.bio_session.lock()); + return Err(e); + } + Ok(()) + } + + pub(crate) fn bio_release(&self) { + bio::release(&mut self.bio_session.lock()); + if let Some(dev) = self.bio_dev.lock().as_ref() { + dev.wake(); + } + } + + pub(crate) fn queue_enrolment(this: Arc) { + if workqueue::system() + .enqueue::, ENROL_WORK_ID>(this.clone()) + .is_err() + { + this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + } + } + + pub(crate) fn queue_verify(this: Arc) { + if workqueue::system() + .enqueue::, VERIFY_WORK_ID>(this.clone()) + .is_err() + { + this.finish_verify( + bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), + [0u8; bio::TOKEN_LEN], + ); + } + } + + fn finish_verify(&self, outcome: bio::VerifyOutcome, token_bytes: [u8; bio::TOKEN_LEN]) { + let filed = bio::verify_finish(&mut self.bio_session.lock(), outcome, token_bytes); + if filed { + self.bio_wake(); + } + } + + fn mint_token_bytes(&self) -> Option<[u8; bio::TOKEN_LEN]> { + let mut bytes = [0u8; bio::TOKEN_LEN]; + for chunk in bytes.chunks_mut(4) { + match self.get_entropy_word() { + Ok(word) => chunk.copy_from_slice(&word.to_le_bytes()[..chunk.len()]), + Err(_) => { + return None; + } + } + } + Some(bytes) + } + + pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { + if cmd == bio::IOC_ATTEST { + return self.bio_attest(arg); + } + let handled = { + let mut session = self.bio_session.lock(); + let index = self.bio_index.lock(); + let mut ctx = bio::Context { + session: &mut session, + index: &index, + sensor_present: self.sensor_present.load(Relaxed), + }; + bio::ioctl(&mut ctx, cmd, arg)? + }; + + // 0x57 sent with the session lock released (it waits on a mailbox reply) + if let Some(identity) = handled.delete_identity { + self.delete_identity(&identity)?; + } + + let mut delete_all_err: Option = None; + for identity in handled.delete_identities.iter() { + if let Err(e) = self.delete_identity(identity) { + delete_all_err = delete_all_err.or(Some(e)); + } + } + if let Some(e) = delete_all_err { + return Err(e); + } + + if handled.wake { + self.bio_wake(); + } + Ok(handled) + } + + fn bio_attest(&self, arg: usize) -> Result { + let user = kernel::uaccess::UserPtr::from_addr(arg); + let req: bio::Attest = + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .reader() + .read()?; + let (sig, pubk) = self.refkey_attest_sign(&req.challenge)?; + if sig.len() > bio::ATTEST_SIG_MAX || pubk.len() != bio::ATTEST_PUB_LEN { + return Err(EIO); + } + let mut out = bio::Attest { + sig_len: sig.len() as u32, + challenge: req.challenge, + public: [0u8; bio::ATTEST_PUB_LEN], + signature: [0u8; bio::ATTEST_SIG_MAX], + reserved: [0u8; 3], + }; + out.public.copy_from_slice(&pubk); + out.signature[..sig.len()].copy_from_slice(&sig); + kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) + .writer() + .write(&out)?; + Ok(bio::Handled { + ret: 0, + wake: false, + start_enrol: false, + start_verify: false, + delete_identity: None, + delete_identities: KVec::new(), + }) + } + + fn ensure_restored_after(&self, _prepared: bool) { + let restored = self.restore_all_components(); + self.templates_restored.store(restored, Relaxed); + if !restored { + dev_err!( + self.dev, + "matching unavailable this boot: restore did not complete, enclave holds no template (every match refused 0x1). Not the sensor or the finger; on-disk enrolments intact\n" + ); + return; + } + + if self.keybag_designated.load(Relaxed) { + if let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) { + let special = user.special_handle(); + if let Ok(keybag::State::Present(stored)) = keybag::read(keybag::Slot::Identity) { + let _ = self.sks_step(crate::sks::SKS_LOCK_STATE_NAME, |healthy| { + self.sks_req_unlock_special(special, stored.secret(), healthy) + }); + } + } + } + + // enclave's match arm asserts a non-empty ACM context (<= 32 bytes) + if let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) { + self.establish_scrd_match_context(user); + } + } + + fn cold_match_continue( + &self, + source: crate::sks::KeyBagHandle, + uuid: [u8; keybag::UUID_LEN], + ) -> bool { + if !self.keybag_designated.load(Relaxed) { + return false; + } + + let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return false; + }; + let special = user.special_handle(); + self.log_identity_count(c"before the cold-match preparation"); + + let uuid_ok = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)); + match uuid_ok { + Some(got) if got == uuid => {} + Some(_) => { + return false; + } + None => { + return false; + } + } + + if self.sks_send(self.sks_req_unload_keybag(source)).is_none() { + return false; + } + + true + } + + pub(crate) fn sep_random(&self, buf: &mut [u8]) -> Result<()> { + for chunk in buf.chunks_mut(4) { + let word = self.get_entropy_word()?; + let bytes = word.to_le_bytes(); + let take = chunk.len(); + chunk.copy_from_slice(&bytes[..take]); + } + Ok(()) + } + + pub(crate) fn wrapped_from_copy_reply( + &self, + out: &SksOutcome, + _from: &CStr, + ) -> Option> { + let body = self.sks_report_response(c"COPY_KEYBAG", out)?; + if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { + return None; + } + let (blob, _) = image::read_blob(body, 4)?; + if blob.is_empty() { + return None; + } + let mut copy = KVec::new(); + if copy.extend_from_slice(blob, GFP_KERNEL).is_err() { + return None; + } + Some(copy) + } + + fn prove_restore(&self) -> bool { + let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { + return false; + }; + let op = crate::sbio::sbio_free_identity_count(user); + let Some(reply) = self.sbio_expect_ok(&op) else { + return false; + }; + if reply.len() != crate::sbio::SBIO_FREE_COUNT_REPLY_LEN { + return false; + } + let count = u32::from_le_bytes([reply[0], reply[1], reply[2], reply[3]]); + if count > crate::sbio::SBIO_FREE_COUNT_MAX { + return false; + } + if count == crate::sbio::SBIO_FREE_COUNT_MAX { + return true; + } + true + } + + fn log_identity_count(&self, _when: &CStr) -> Option { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + let records = crate::sbio::IdentityRecords::new(&reply)?; + Some(records.count()) + } + + fn enclave_lists(&self, uuid: &[u8; bio::UUID_LEN]) -> Option { + let op = crate::sbio::sbio_list_identities(); + let SbioOutcome::Ok(reply) = self.sbio_call(&op) else { + return None; + }; + let records = crate::sbio::IdentityRecords::new(&reply)?; + Some(records.lists_uuid(uuid)) + } + + fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result<()> { + if self.enclave_lists(identity.uuid()) == Some(false) { + self.forget_identity(identity.uuid(), c"a host entry the enclave never had"); + return Ok(()); + } + + let op = crate::sbio::sbio_delete_identity(identity); + + if self.sbio_expect_ok(&op).is_none() { + dev_err!( + self.dev, + "bio: 0x57 failed, status above; host index left unchanged (forgetting a template the enclave still holds would report a deletion that did not happen)\n" + ); + return Err(EIO); + } + + self.forget_identity(identity.uuid(), c"removed from the enclave"); + Ok(()) + } + + fn forget_identity(&self, uuid: &[u8; bio::UUID_LEN], why: &CStr) { + let persisted = { + let mut index = self.bio_index.lock(); + index.remove(uuid); + self.with_host_store(|store| index.persist(store)) + }; + match persisted { + Some(Ok(())) => {}, + _ => dev_warn!( + self.dev, + "bio: {} ({}) but the host index could not be written; correct in memory this boot, rebuilt at next attach\n", + Hex(uuid), + why + ), + } + } + + pub(crate) fn bio_ready(&self) -> bool { + self.bio_session.lock().ready() + } +} + +pub(crate) struct SbioOp { + opcode: u16, + payload: [u8; SBIO_MAX_PAYLOAD], + payload_len: usize, + name: &'static CStr, +} + +const SBIO_MAX_PAYLOAD: usize = 0x97; + +pub(crate) const SBIO_IMAGE_INIT_LEN: usize = 0x97; +static_assert!(SBIO_IMAGE_INIT_LEN <= SBIO_MAX_PAYLOAD); + +const fn pad_payload(src: &[u8]) -> [u8; SBIO_MAX_PAYLOAD] { + let mut out = [0u8; SBIO_MAX_PAYLOAD]; + let mut i = 0; + while i < src.len() { + out[i] = src[i]; + i += 1; + } + out +} + +impl SbioOp { + pub(crate) fn opcode(&self) -> u16 { + self.opcode + } + pub(crate) fn payload(&self) -> &[u8] { + &self.payload[..self.payload_len] + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +pub(crate) const SBIO_PROTOCOL_GENERATION: u32 = 1; + +const OP_SBIO_REGISTER_SENSOR: u16 = 0x80; +pub(crate) fn sbio_register_sensor(id: &crate::sensor::Identifier) -> SbioOp { + SbioOp { + opcode: OP_SBIO_REGISTER_SENSOR, + payload: pad_payload(&id.value().to_le_bytes()), + payload_len: 2, + name: c"REGISTER_SENSOR", + } +} + +pub(crate) const SBIO_SIGNAL_QUALITY: u32 = 0; + +const OP_SBIO_COVERAGE_PARAMS: u16 = 0x5d; +pub(crate) fn sbio_coverage_params() -> SbioOp { + SbioOp { + opcode: OP_SBIO_COVERAGE_PARAMS, + payload: pad_payload(&SBIO_SIGNAL_QUALITY.to_le_bytes()), + payload_len: 4, + name: c"COVERAGE_PARAMS", + } +} + +const OP_SBIO_OPERATION_PARAMS: u16 = 0x5c; + +pub(crate) const SBIO_OPERATION_PARAMS_LEN: usize = match SBIO_PROTOCOL_GENERATION { + 1 => 4, + 6 => 8, + _ => 0, +}; +static_assert!(SBIO_OPERATION_PARAMS_LEN != 0); +static_assert!(SBIO_OPERATION_PARAMS_LEN <= SBIO_MAX_PAYLOAD); + +pub(crate) fn sbio_operation_params() -> SbioOp { + SbioOp { + opcode: OP_SBIO_OPERATION_PARAMS, + payload: pad_payload(&[]), + payload_len: SBIO_OPERATION_PARAMS_LEN, + name: c"OPERATION_PARAMS", + } +} + +const OP_SBIO_TRANSPARENT_CHANNEL: u16 = 0x6a; +pub(crate) fn sbio_transparent_channel() -> SbioOp { + SbioOp { + opcode: OP_SBIO_TRANSPARENT_CHANNEL, + payload: pad_payload(&[]), + payload_len: 0, + name: c"TRANSPARENT_CHANNEL", + } +} + +const OP_SBIO_COMPLETE_INIT: u16 = 0x01; +pub(crate) fn sbio_complete_init( + patch: crate::PatchLoaded, + params: crate::ParametersApplied, +) -> SbioOp { + let (_, _) = (patch, params); + let mut body = [0u8; 8]; + body[0..4].copy_from_slice(&1u32.to_le_bytes()); + body[4..8].copy_from_slice(&1u32.to_le_bytes()); + SbioOp { + opcode: OP_SBIO_COMPLETE_INIT, + payload: pad_payload(&body), + payload_len: 8, + name: c"COMPLETE_INIT", + } +} + +const CONTEXT_SCOPE_SYSTEM: i32 = -1; +static_assert!(CONTEXT_SCOPE_SYSTEM != 0); +static_assert!(CONTEXT_SCOPE_SYSTEM < 0); + +#[derive(Clone, Copy)] +pub(crate) struct ContextScope(i32); + +impl ContextScope { + pub(crate) const SYSTEM: ContextScope = ContextScope(CONTEXT_SCOPE_SYSTEM); + + pub(crate) fn user(id: UserId) -> ContextScope { + ContextScope(id.value()) + } + + pub(crate) fn value(&self) -> i32 { + self.0 + } +} + +const OP_SBIO_CONTEXT_STATE: u16 = 0x6b; +pub(crate) fn sbio_context_state() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CONTEXT_STATE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CONTEXT_STATE", + } +} + +// 0x2d destroys the catacomb; only send with zero existing identities +pub(crate) struct NoExistingCatacomb { + _seal: (), +} + +impl NoExistingCatacomb { + pub(crate) fn from_zero_identities(count: usize) -> Option { + match count { + 0 => Some(NoExistingCatacomb { _seal: () }), + _ => None, + } + } +} + +const OP_SBIO_SELECT_CONTEXT: u16 = 0x2d; +pub(crate) fn sbio_select_context(scope: ContextScope, proof: &NoExistingCatacomb) -> SbioOp { + let NoExistingCatacomb { _seal: () } = proof; + SbioOp { + opcode: OP_SBIO_SELECT_CONTEXT, + payload: pad_payload(&scope.value().to_le_bytes()), + payload_len: 4, + name: c"SELECT_CONTEXT", + } +} + +pub(crate) const SBIO_PROTECTED_CONFIG_LEN: usize = 32; + +const OP_SBIO_PROTECTED_CONFIG: u16 = 0x2b; +pub(crate) fn sbio_protected_config(id: UserId) -> SbioOp { + SbioOp { + opcode: OP_SBIO_PROTECTED_CONFIG, + payload: pad_payload(&id.value().to_le_bytes()), + payload_len: 4, + name: c"PROTECTED_CONFIG", + } +} + +const OP_SBIO_BEGIN_ENROL: u16 = 0x03; + +pub(crate) const BE_AUTH_TYPE_ACM_CONTEXT: u32 = 0; + +pub(crate) const SBIO_BEGIN_ENROL_LEN: usize = 0x44; +static_assert!(SBIO_BEGIN_ENROL_LEN <= SBIO_MAX_PAYLOAD); + +const BE_FLAGS: usize = 0; +const BE_USER_ID: usize = 4; +const BE_AUTH_TYPE: usize = 8; +const BE_TOKEN_LEN: usize = 12; +const BE_TOKEN: usize = 16; +static_assert!(BE_TOKEN + SKS_AUTH_TOKEN_LEN <= SBIO_BEGIN_ENROL_LEN); + +pub(crate) fn sbio_begin_enrol( + user: UserId, + auth_type: u32, + token: &[u8; SKS_AUTH_TOKEN_LEN], +) -> SbioOp { + let mut body = [0u8; SBIO_BEGIN_ENROL_LEN]; + body[BE_FLAGS..BE_FLAGS + 4].copy_from_slice(&0u32.to_le_bytes()); + body[BE_USER_ID..BE_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + body[BE_AUTH_TYPE..BE_AUTH_TYPE + 4].copy_from_slice(&auth_type.to_le_bytes()); + body[BE_TOKEN_LEN..BE_TOKEN_LEN + 4] + .copy_from_slice(&(SKS_AUTH_TOKEN_LEN as u32).to_le_bytes()); + body[BE_TOKEN..BE_TOKEN + SKS_AUTH_TOKEN_LEN].copy_from_slice(token); + SbioOp { + opcode: OP_SBIO_BEGIN_ENROL, + payload: pad_payload(&body), + payload_len: SBIO_BEGIN_ENROL_LEN, + name: c"BEGIN_ENROL", + } +} + +const OP_SBIO_MODULE_CHALLENGE: u16 = 0x31; +pub(crate) fn sbio_module_challenge() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MODULE_CHALLENGE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MODULE_CHALLENGE", + } +} + +pub(crate) const SBIO_MODULE_CHALLENGE_LEN: usize = 0x40; + +const OP_SBIO_MODULE_COMMIT: u16 = 0x32; +pub(crate) fn sbio_module_commit(reply: &[u8]) -> Option { + if reply.is_empty() || reply.len() > SBIO_MAX_PAYLOAD { + return None; + } + Some(SbioOp { + opcode: OP_SBIO_MODULE_COMMIT, + payload: pad_payload(reply), + payload_len: reply.len(), + name: c"MODULE_COMMIT", + }) +} + +const OP_SBIO_LOAD_CALIBRATION: u16 = 0x5b; + +pub(crate) const SBIO_CALIBRATION_SOURCE: u32 = 3; + +pub(crate) struct SbioCalibration(KVec); + +impl SbioCalibration { + pub(crate) fn new(blob: &crate::CalibrationBlob) -> Result { + let bytes = blob.bytes(); + let mut body = KVec::new(); + body.extend_from_slice(&SBIO_CALIBRATION_SOURCE.to_le_bytes(), GFP_KERNEL)?; + body.extend_from_slice(bytes, GFP_KERNEL)?; + Ok(SbioCalibration(body)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_CALIBRATION + } + + pub(crate) fn payload(&self) -> &[u8] { + &self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_CALIBRATION" + } +} + +const OP_SBIO_UPDATE_DEVICE_LIST: u16 = 0x7b; +pub(crate) fn sbio_update_device_list() -> SbioOp { + SbioOp { + opcode: OP_SBIO_UPDATE_DEVICE_LIST, + payload: pad_payload(&[]), + payload_len: 0, + name: c"UPDATE_DEVICE_LIST", + } +} + +pub(crate) const SBIO_MATCH_POLICY_LEN: usize = 2; + +const OP_SBIO_MATCH_POLICY: u16 = 0x1f; +pub(crate) fn sbio_match_policy() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MATCH_POLICY, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MATCH_POLICY", + } +} + +pub(crate) const SBIO_ENROL_RESULT_LEN: usize = 0xc98; + +const ER_PROGRESS: usize = 0x004; +const ER_HAS_TEMPLATE: usize = 0x006; +const ER_COMPLETE: usize = 0xbfe; + +static_assert!(ER_PROGRESS < SBIO_ENROL_RESULT_LEN); +static_assert!(ER_HAS_TEMPLATE + 4 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_COMPLETE + 4 <= SBIO_ENROL_RESULT_LEN); +static_assert!(ER_COMPLETE > ER_HAS_TEMPLATE + 4); + +pub(crate) const ER_IDENTITY_LEN: usize = 4 + IDENTITY_UUID_LEN; +static_assert!(ER_IDENTITY_LEN == 20); +static_assert!(ER_IDENTITY_LEN <= SBIO_ENROL_RESULT_LEN); + +pub(crate) fn enrol_identity_candidates( + bytes: &[u8], + user_id: i32, + mut visit: impl FnMut(usize, [u8; IDENTITY_UUID_LEN]), +) { + if bytes.len() < ER_IDENTITY_LEN { + return; + } + let wanted = user_id.to_le_bytes(); + for at in 0..=bytes.len() - ER_IDENTITY_LEN { + if bytes[at..at + 4] != wanted { + continue; + } + let mut uuid = [0u8; IDENTITY_UUID_LEN]; + uuid.copy_from_slice(&bytes[at + 4..at + ER_IDENTITY_LEN]); + if uuid.iter().all(|&b| b == 0) { + continue; + } + visit(at, uuid); + } +} + +pub(crate) struct EnrolmentResult { + progress_raw: u8, + has_template: u32, + complete: u32, +} + +impl EnrolmentResult { + pub(crate) fn parse(bytes: &[u8]) -> Option { + if bytes.len() < SBIO_ENROL_RESULT_LEN { + return None; + } + Some(EnrolmentResult { + progress_raw: bytes[ER_PROGRESS], + has_template: u32::from_le_bytes([ + bytes[ER_HAS_TEMPLATE], + bytes[ER_HAS_TEMPLATE + 1], + bytes[ER_HAS_TEMPLATE + 2], + bytes[ER_HAS_TEMPLATE + 3], + ]), + complete: u32::from_le_bytes([ + bytes[ER_COMPLETE], + bytes[ER_COMPLETE + 1], + bytes[ER_COMPLETE + 2], + bytes[ER_COMPLETE + 3], + ]), + }) + } + + pub(crate) fn progress_percent(&self) -> u32 { + (u32::from(self.progress_raw) * 100 + 127) / 255 + } + + pub(crate) fn has_template(&self) -> bool { + self.has_template != 0 + } + + pub(crate) fn complete(&self) -> bool { + self.complete != 0 + } +} + +#[derive(Clone, Copy)] +pub(crate) struct CatacombUser(i32); + +impl CatacombUser { + pub(crate) const MASTER: CatacombUser = CatacombUser(-1); + pub(crate) const OWNER: CatacombUser = CatacombUser(501); + + pub(crate) fn enrolling(user: UserId) -> CatacombUser { + CatacombUser(user.value()) + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} +static_assert!(CatacombUser::MASTER.value() == -1); +static_assert!(CatacombUser::OWNER.value() == 501); +static_assert!(CatacombUser::MASTER.value() != CatacombUser::OWNER.value()); +static_assert!(CatacombUser::MASTER.value() < 0); + +pub(crate) const SBIO_SAVED_USER_ID_AT: usize = 8; +static_assert!(SBIO_SAVED_USER_ID_AT + 4 <= SBIO_SAVED_MIN); + +pub(crate) const SBIO_SAVE_SELECTOR_LEN: usize = 24; +const SS_USER_ID: usize = 0; +static_assert!(SBIO_SAVE_SELECTOR_LEN <= SBIO_MAX_PAYLOAD); + +pub(crate) struct SaveSelector([u8; SBIO_SAVE_SELECTOR_LEN]); + +impl SaveSelector { + pub(crate) fn new(user: CatacombUser) -> SaveSelector { + let mut out = [0u8; SBIO_SAVE_SELECTOR_LEN]; + out[SS_USER_ID..SS_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + SaveSelector(out) + } + + pub(crate) fn bytes(&self) -> &[u8; SBIO_SAVE_SELECTOR_LEN] { + &self.0 + } +} + +pub(crate) const SBIO_STATUS_COLD_TRANSITION: u32 = 0x8002; +static_assert!(SBIO_STATUS_COLD_TRANSITION != 0); + +// 0x101 from a 0x6d load = already active; a success answer, not an error +pub(crate) const SBIO_STATUS_ALREADY_ACTIVE: u32 = 0x101; +static_assert!(SBIO_STATUS_ALREADY_ACTIVE != 0); +static_assert!(SBIO_STATUS_ALREADY_ACTIVE != SBIO_STATUS_COLD_TRANSITION); + +pub(crate) const COMPONENT_STATE_COLD: u32 = 0x1; +pub(crate) const COMPONENT_STATE_ACTIVE: u32 = 0x2; +static_assert!(COMPONENT_STATE_COLD != COMPONENT_STATE_ACTIVE); + +pub(crate) const COMPONENT_PAIR_LEN: usize = 8; + +pub(crate) struct ComponentStates<'a>(&'a [u8]); + +impl<'a> ComponentStates<'a> { + pub(crate) fn new(reply: &'a [u8]) -> ComponentStates<'a> { + ComponentStates(reply) + } + + pub(crate) fn count(&self) -> usize { + self.0.len() / COMPONENT_PAIR_LEN + } + + pub(crate) fn pair(&self, i: usize) -> Option<(i32, u32)> { + let at = i.checked_mul(COMPONENT_PAIR_LEN)?; + let bytes = self.0.get(at..at + COMPONENT_PAIR_LEN)?; + Some(( + i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + )) + } + + pub(crate) fn state_for(&self, id: i32) -> Option { + (0..self.count()) + .filter_map(|i| self.pair(i)) + .find(|(pair_id, _)| *pair_id == id) + .map(|(_, state)| state) + } +} + +pub(crate) enum ComponentAction { + AlreadyActive, + Load, + Unsupported, +} + +pub(crate) fn component_action(state: u32) -> ComponentAction { + if state & COMPONENT_STATE_ACTIVE != 0 { + ComponentAction::AlreadyActive + } else if state & COMPONENT_STATE_COLD != 0 { + ComponentAction::Load + } else { + ComponentAction::Unsupported + } +} + +const OP_SBIO_SAVE_LOCKOUT: u16 = 0x70; +pub(crate) fn sbio_save_lockout() -> SbioOp { + SbioOp { + opcode: OP_SBIO_SAVE_LOCKOUT, + payload: pad_payload(&[]), + payload_len: 0, + name: c"SAVE_LOCKOUT", + } +} + +pub(crate) struct SbioLoadLockout<'a>(&'a [u8]); + +impl<'a> SbioLoadLockout<'a> { + pub(crate) fn new(blob: &'a [u8]) -> Option> { + if blob.is_empty() || blob.len() > SBIO_SAVED_MAX { + return None; + } + Some(SbioLoadLockout(blob)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_LOCKOUT + } + + pub(crate) fn payload(&self) -> &[u8] { + self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_LOCKOUT" + } +} +const OP_SBIO_LOAD_LOCKOUT: u16 = 0x71; + +const OP_SBIO_SAVE_CATACOMB: u16 = 0x6c; +pub(crate) fn sbio_save_catacomb(selector: &SaveSelector) -> SbioOp { + SbioOp { + opcode: OP_SBIO_SAVE_CATACOMB, + payload: pad_payload(selector.bytes()), + payload_len: SBIO_SAVE_SELECTOR_LEN, + name: c"SAVE_CATACOMB", + } +} + +const OP_SBIO_CONFIRM_SAVE: u16 = 0x37; +pub(crate) fn sbio_confirm_save(selector: &SaveSelector) -> SbioOp { + SbioOp { + opcode: OP_SBIO_CONFIRM_SAVE, + payload: pad_payload(selector.bytes()), + payload_len: SBIO_SAVE_SELECTOR_LEN, + name: c"CONFIRM_SAVE", + } +} + +pub(crate) const SBIO_SAVED_MIN: usize = 0x22; +pub(crate) const SBIO_SAVED_MAX: usize = 0x4b000; +static_assert!(SBIO_SAVED_MIN < SBIO_SAVED_MAX); +static_assert!(SBIO_SAVED_MAX as u32 <= transfer::MAX_TRANSACTION); + +pub(crate) struct SbioLoadCatacomb<'a>(&'a [u8]); + +impl<'a> SbioLoadCatacomb<'a> { + pub(crate) fn new(blob: &'a [u8]) -> Option> { + if blob.len() < SBIO_SAVED_MIN || blob.len() > SBIO_SAVED_MAX { + return None; + } + Some(SbioLoadCatacomb(blob)) + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_LOAD_CATACOMB + } + + pub(crate) fn payload(&self) -> &[u8] { + self.0 + } + + pub(crate) fn name(&self) -> &'static CStr { + c"LOAD_CATACOMB" + } +} +const OP_SBIO_LOAD_CATACOMB: u16 = 0x6d; + +pub(crate) const IDENTITY_RECORD_LEN: usize = 40; +const IR_SELECTOR: usize = 20; +const IR_SELECTOR_BUILTIN: u32 = 1; +static_assert!(IR_SELECTOR == IDENTITY_V1_LEN); +static_assert!(IR_SELECTOR + 4 <= IDENTITY_RECORD_LEN); + +const OP_SBIO_LIST_IDENTITIES: u16 = 0x7a; +pub(crate) fn sbio_list_identities() -> SbioOp { + SbioOp { + opcode: OP_SBIO_LIST_IDENTITIES, + payload: pad_payload(&[]), + payload_len: 0, + name: c"LIST_IDENTITIES", + } +} + +pub(crate) struct IdentityRecords<'a>(&'a [u8]); + +impl<'a> IdentityRecords<'a> { + pub(crate) fn new(reply: &'a [u8]) -> Option> { + if reply.len() % IDENTITY_RECORD_LEN != 0 { + return None; + } + Some(IdentityRecords(reply)) + } + + pub(crate) fn count(&self) -> usize { + self.0.len() / IDENTITY_RECORD_LEN + } + + pub(crate) fn record(&self, i: usize) -> Option<(IdentityV1, bool)> { + let at = i.checked_mul(IDENTITY_RECORD_LEN)?; + let bytes = self.0.get(at..at + IDENTITY_RECORD_LEN)?; + let user_id = i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let mut uuid = [0u8; IDENTITY_UUID_LEN]; + uuid.copy_from_slice(&bytes[4..IDENTITY_V1_LEN]); + let selector = u32::from_le_bytes([ + bytes[IR_SELECTOR], + bytes[IR_SELECTOR + 1], + bytes[IR_SELECTOR + 2], + bytes[IR_SELECTOR + 3], + ]); + Some(( + IdentityV1 { user_id, uuid }, + selector == IR_SELECTOR_BUILTIN, + )) + } + + pub(crate) fn lists_uuid(&self, uuid: &[u8; IDENTITY_UUID_LEN]) -> bool { + (0..self.count()) + .filter_map(|i| self.record(i)) + .any(|(identity, _)| identity.uuid == *uuid) + } + + pub(crate) fn each_built_in_for(&self, user_id: i32, mut visit: impl FnMut(IdentityV1)) { + for i in 0..self.count() { + if let Some((identity, built_in)) = self.record(i) { + if built_in && identity.user_id == user_id { + visit(identity); + } + } + } + } +} + +const FIC_DEVICE_BUILTIN: u32 = 1; +pub(crate) const SBIO_FREE_COUNT_LEN: usize = 4 + 20; +static_assert!(SBIO_FREE_COUNT_LEN == SBIO_SAVE_SELECTOR_LEN); +pub(crate) const SBIO_FREE_COUNT_REPLY_LEN: usize = 4; +pub(crate) const SBIO_FREE_COUNT_MAX: u32 = 3; + +const OP_SBIO_FREE_IDENTITY_COUNT: u16 = 0x38; +pub(crate) fn sbio_free_identity_count(user: UserId) -> SbioOp { + let mut body = [0u8; SBIO_FREE_COUNT_LEN]; + body[0..4].copy_from_slice(&user.value().to_le_bytes()); + body[4..8].copy_from_slice(&FIC_DEVICE_BUILTIN.to_le_bytes()); + SbioOp { + opcode: OP_SBIO_FREE_IDENTITY_COUNT, + payload: pad_payload(&body), + payload_len: SBIO_FREE_COUNT_LEN, + name: c"FREE_IDENTITY_COUNT", + } +} + +const OP_SBIO_DELETE_IDENTITY: u16 = 0x57; +pub(crate) fn sbio_delete_identity(identity: &IdentityV1) -> SbioOp { + SbioOp { + opcode: OP_SBIO_DELETE_IDENTITY, + payload: pad_payload(&identity.wire()), + payload_len: IDENTITY_V1_LEN, + name: c"DELETE_IDENTITY", + } +} + +const OP_SBIO_MATCH_RESULT: u16 = 0x09; +pub(crate) fn sbio_match_result() -> SbioOp { + SbioOp { + opcode: OP_SBIO_MATCH_RESULT, + payload: pad_payload(&[]), + payload_len: 0, + name: c"MATCH_RESULT", + } +} + +pub(crate) const SBIO_MATCH_RESULT_LEN: usize = 0xca2; + +const MR_USER_ID: usize = 0x000; +const MR_IDENTITY: usize = 0x004; +static_assert!(MR_USER_ID + 4 <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_IDENTITY + IDENTITY_UUID_LEN <= SBIO_MATCH_RESULT_LEN); +static_assert!(MR_IDENTITY == MR_USER_ID + 4); +static_assert!(SBIO_MATCH_RESULT_LEN != SBIO_ENROL_RESULT_LEN); + +pub(crate) const IDENTITY_UUID_LEN: usize = 16; +pub(crate) const IDENTITY_V1_LEN: usize = 4 + IDENTITY_UUID_LEN; +static_assert!(IDENTITY_V1_LEN == 0x14); + +#[derive(Clone, Copy)] +pub(crate) struct IdentityV1 { + user_id: i32, + uuid: [u8; IDENTITY_UUID_LEN], +} + +impl IdentityV1 { + pub(crate) fn from_index(user_id: i32, uuid: [u8; IDENTITY_UUID_LEN]) -> IdentityV1 { + IdentityV1 { user_id, uuid } + } + + pub(crate) fn uuid(&self) -> &[u8; IDENTITY_UUID_LEN] { + &self.uuid + } + + pub(crate) fn wire(&self) -> [u8; IDENTITY_V1_LEN] { + let mut out = [0u8; IDENTITY_V1_LEN]; + out[0..4].copy_from_slice(&self.user_id.to_le_bytes()); + out[4..IDENTITY_V1_LEN].copy_from_slice(&self.uuid); + out + } +} + +pub(crate) struct MatchResult { + user_id: i32, + identity: [u8; IDENTITY_UUID_LEN], +} + +impl MatchResult { + pub(crate) fn parse(bytes: &[u8]) -> Option { + if bytes.len() < SBIO_MATCH_RESULT_LEN { + return None; + } + let mut identity = [0u8; IDENTITY_UUID_LEN]; + identity.copy_from_slice(&bytes[MR_IDENTITY..MR_IDENTITY + IDENTITY_UUID_LEN]); + Some(MatchResult { + user_id: i32::from_le_bytes([ + bytes[MR_USER_ID], + bytes[MR_USER_ID + 1], + bytes[MR_USER_ID + 2], + bytes[MR_USER_ID + 3], + ]), + identity, + }) + } + + pub(crate) fn identity_uuid(&self) -> &[u8; IDENTITY_UUID_LEN] { + &self.identity + } + + pub(crate) fn matches(&self, user: UserId) -> bool { + self.user_id == user.value() + } +} + +const OP_SBIO_IMAGE_CLEANUP: u16 = 0x22; +pub(crate) fn sbio_image_cleanup() -> SbioOp { + SbioOp { + opcode: OP_SBIO_IMAGE_CLEANUP, + payload: pad_payload(&[]), + payload_len: 0, + name: c"IMAGE_CLEANUP", + } +} + +const OP_SBIO_CANCEL: u16 = 0x05; +pub(crate) fn sbio_cancel_operation() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CANCEL, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CANCEL_OPERATION", + } +} + +const OP_SBIO_CLEAR_STATE: u16 = 0x19; +pub(crate) fn sbio_clear_state() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CLEAR_STATE, + payload: pad_payload(&[]), + payload_len: 0, + name: c"CLEAR_STATE", + } +} + +const OP_SBIO_REGISTER_SERIAL: u16 = 0x17; +pub(crate) fn sbio_register_sensor_serial(serial: &crate::sensor::SensorSerial) -> SbioOp { + SbioOp { + opcode: OP_SBIO_REGISTER_SERIAL, + payload: pad_payload(serial.bytes()), + payload_len: crate::sensor::SENSOR_SERIAL_LEN, + name: c"REGISTER_SERIAL", + } +} + +const OP_SBIO_DIAGNOSTICS: u16 = 0x63; +pub(crate) const fn sbio_diagnostics() -> SbioOp { + SbioOp { + opcode: OP_SBIO_DIAGNOSTICS, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"DIAGNOSTICS", + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ImagePurpose { + Enrolment, + Matching, +} + +impl ImagePurpose { + const fn offset(self) -> usize { + match self { + ImagePurpose::Matching => 0x12, + ImagePurpose::Enrolment => 0x13, + } + } + + const fn last_image_offset(self) -> usize { + match self { + ImagePurpose::Enrolment => 0x08, + ImagePurpose::Matching => 0x09, + } + } +} + +static_assert!(ImagePurpose::Enrolment.offset() != ImagePurpose::Matching.offset()); +static_assert!(ImagePurpose::Matching.offset() == 0x12); +static_assert!(ImagePurpose::Enrolment.offset() == 0x13); +static_assert!(ImagePurpose::Enrolment.last_image_offset() == 0x08); +static_assert!(ImagePurpose::Matching.last_image_offset() == 0x09); +static_assert!( + ImagePurpose::Enrolment.last_image_offset() != ImagePurpose::Matching.last_image_offset() +); +static_assert!(ImagePurpose::Matching.last_image_offset() < ImagePurpose::Matching.offset()); + +#[derive(Clone, Copy)] +pub(crate) struct UserId(i32); + +impl UserId { + pub(crate) fn new(value: i32) -> Option { + if value > 0 { + Some(UserId(value)) + } else { + None + } + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +const IPI_FIRST_IMAGE: usize = 0x07; +const IPI_TIMESTAMP: usize = 0x40; +const IPI_CAPTURE_COUNTER: usize = 0x48; +const IPI_DEVICE_KIND: usize = 0x4f; +const IPI_USER_ID: usize = 0x67; + +const IPI_DEVICE_BUILTIN: u32 = 1; + +static_assert!(IPI_USER_ID + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_DEVICE_KIND + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_TIMESTAMP + 8 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_CAPTURE_COUNTER + 4 <= SBIO_MAX_PAYLOAD); +static_assert!(IPI_FIRST_IMAGE < ImagePurpose::Matching.offset()); +static_assert!(ImagePurpose::Enrolment.offset() < IPI_TIMESTAMP); + +const OP_SBIO_PREPARE_IMAGE: u16 = 0x23; +pub(crate) fn sbio_prepare_image_processing() -> SbioOp { + SbioOp { + opcode: OP_SBIO_PREPARE_IMAGE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"PREPARE_IMAGE_PROCESSING", + } +} + +const OP_SBIO_IMAGE_INIT: u16 = 0x06; +pub(crate) fn sbio_image_processing_init( + purpose: ImagePurpose, + first_image: bool, + last_image: bool, + capture_counter: u32, + user: UserId, + monotonic_ns: u64, +) -> SbioOp { + let mut payload = [0u8; SBIO_MAX_PAYLOAD]; + + if first_image { + payload[IPI_FIRST_IMAGE] = 1; + } + if last_image { + payload[purpose.last_image_offset()] = 1; + } + payload[purpose.offset()] = 1; + + // ns*24/1000 split to dodge u64 overflow and the kernel's absent __udivti3 + let scaled = (monotonic_ns / 1000) * 24 + ((monotonic_ns % 1000) * 24) / 1000; + payload[IPI_TIMESTAMP..IPI_TIMESTAMP + 8].copy_from_slice(&scaled.to_le_bytes()); + payload[IPI_CAPTURE_COUNTER..IPI_CAPTURE_COUNTER + 4] + .copy_from_slice(&capture_counter.to_le_bytes()); + payload[IPI_DEVICE_KIND..IPI_DEVICE_KIND + 4] + .copy_from_slice(&IPI_DEVICE_BUILTIN.to_le_bytes()); + payload[IPI_USER_ID..IPI_USER_ID + 4].copy_from_slice(&user.value().to_le_bytes()); + + SbioOp { + opcode: OP_SBIO_IMAGE_INIT, + payload, + payload_len: SBIO_IMAGE_INIT_LEN, + name: c"IMAGE_PROCESSING_INIT", + } +} + +const OP_SBIO_RELAY_CAPTURE: u16 = 0x65; + +pub(crate) struct SbioRelay<'a> { + capture: &'a crate::sensor::Capture, +} + +impl<'a> SbioRelay<'a> { + pub(crate) fn new(capture: &'a crate::sensor::Capture) -> SbioRelay<'a> { + SbioRelay { capture } + } + + pub(crate) fn opcode(&self) -> u16 { + OP_SBIO_RELAY_CAPTURE + } + + pub(crate) fn payload(&self) -> &[u8] { + self.capture.bytes() + } + + pub(crate) fn name(&self) -> &'static CStr { + c"RELAY_CAPTURE" + } +} + +const OP_SBIO_ASSESSMENT: u16 = 0x07; +pub(crate) fn sbio_image_assessment() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ASSESSMENT, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"IMAGE_ASSESSMENT", + } +} + +const OP_SBIO_ENROL_RESULT: u16 = 0x08; +pub(crate) fn sbio_enrolment_result() -> SbioOp { + SbioOp { + opcode: OP_SBIO_ENROL_RESULT, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"ENROLMENT_RESULT", + } +} + +pub(crate) const ASSESS_MIN_LEN: usize = 0x89; +pub(crate) const ASSESS_USABLE_MATCH: usize = 0x06; +pub(crate) const ASSESS_USABLE_ENROL: usize = 0x07; + +static_assert!(ASSESS_USABLE_MATCH < ASSESS_USABLE_ENROL); + +pub(crate) const SBIO_SESSION_SHARE_LEN: usize = 40; + +pub(crate) const SBIO_STATUS_OK: u16 = 0x00; +pub(crate) const SBIO_STATUS_PREREQUISITE: u16 = 0x01; +pub(crate) const SBIO_STATUS_16: u16 = 0x16; +static_assert!(SBIO_STATUS_PREREQUISITE != SBIO_STATUS_16); +pub(crate) const SBIO_SESSION_MODE: u32 = 1; +static_assert!(SBIO_SESSION_SHARE_LEN <= SBIO_MAX_PAYLOAD); + +const OP_SBIO_SESSION_SHARE: u16 = 0x15; +pub(crate) fn sbio_request_session_share() -> SbioOp { + SbioOp { + opcode: OP_SBIO_SESSION_SHARE, + payload: pad_payload(&SBIO_SESSION_MODE.to_le_bytes()), + payload_len: 4, + name: c"SESSION_SHARE", + } +} + +const OP_SBIO_COMMIT_SESSION: u16 = 0x16; +pub(crate) fn sbio_commit_session_share(share: &[u8; SBIO_SESSION_SHARE_LEN]) -> SbioOp { + SbioOp { + opcode: OP_SBIO_COMMIT_SESSION, + payload: pad_payload(share), + payload_len: SBIO_SESSION_SHARE_LEN, + name: c"SESSION_COMMIT", + } +} +pub(crate) const SBIO_CHALLENGE_LEN: usize = 64; +static_assert!(SBIO_CHALLENGE_LEN <= SBIO_MAX_PAYLOAD); + +const OP_SBIO_CHALLENGE: u16 = 0x42; +pub(crate) fn sbio_request_challenge() -> SbioOp { + SbioOp { + opcode: OP_SBIO_CHALLENGE, + payload: [0; SBIO_MAX_PAYLOAD], + payload_len: 0, + name: c"SEQUENCE_CHALLENGE", + } +} + +const OP_SBIO_COMMIT_CHALLENGE: u16 = 0x18; +pub(crate) fn sbio_commit_challenge(reply: &[u8; SBIO_CHALLENGE_LEN]) -> SbioOp { + SbioOp { + opcode: OP_SBIO_COMMIT_CHALLENGE, + payload: pad_payload(reply), + payload_len: SBIO_CHALLENGE_LEN, + name: c"SEQUENCE_COMMIT", + } +} +const OP_SBIO_FETCH_PATCH: u16 = 0x5f; + +const SBIO_PATCH_SUBTYPE: u16 = 2; + +pub(crate) fn sbio_fetch_patch() -> SbioOp { + SbioOp { + opcode: OP_SBIO_FETCH_PATCH, + payload: pad_payload(&SBIO_PATCH_SUBTYPE.to_le_bytes()), + payload_len: 2, + name: c"FETCH_SENSOR_PATCH", + } +} + +const fn encode_sbio(opcode: u16, marker: u8, seq: u16) -> Message { + Message { + msg0: (EP_SBIO as u64) + | ((marker as u64) << MSG_TAG_SHIFT) + | ((opcode as u64) << MSG_TYPE_SHIFT) + | ((seq as u64) << 48), + msg1: 0, + } +} + +static_assert!(encode_sbio(0x73, transfer::MARKER_FIRST, 0).msg0 == 0x0000_0073_fc08); + +pub(crate) fn encode_sbio_raw(opcode: u16, seq: u16) -> Option { + Some(encode_sbio(opcode, transfer::MARKER_FIRST, seq)) +} + +pub(crate) fn encode_sbio_next(opcode: u16, seq: u16) -> Option { + Some(encode_sbio(opcode, transfer::MARKER_NEXT, seq)) +} + +pub(crate) fn encode_sbio_continue(cont: &transfer::Continuation) -> Message { + encode_sbio(cont.opcode() as u16, transfer::MARKER_REQUEST, cont.seq()) +} diff --git a/drivers/soc/apple/scrd.rs b/drivers/soc/apple/scrd.rs new file mode 100644 index 00000000000000..acdcdab12db658 --- /dev/null +++ b/drivers/soc/apple/scrd.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! SEP credential endpoint (SCRD, EP 0x0a) wire protocol. + +use crate::proto::*; +use crate::sks::SKS_AUTH_TOKEN_LEN; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +pub(crate) const SCRD_ACM_HANDLE_LEN: usize = SKS_AUTH_TOKEN_LEN; + +pub(crate) const SCRD_REQUEST_COMMAND: u8 = 1; + +const SCRD_MAGIC: [u8; 4] = *b"DRCS"; + +const SCRD_CMD_INITIALIZE: u8 = 0x0a; +const SCRD_CMD_CONTEXT_CREATE_TRACKED: u8 = 0x24; +const SCRD_CMD_CONTEXT_EXTERNALIZE: u8 = 0x13; +const SCRD_CMD_VERIFY_POLICY: u8 = 0x03; + +const SCRD_INIT_LOG_LEVEL: u8 = 0x28; + +const SCRD_POLICY_TOUCHID_ENROLLMENT: &[u8] = b"TouchIdEnrollment"; + +const SCRD_MAX_LOGICAL: usize = 64; +static_assert!( + 8 + SCRD_ACM_HANDLE_LEN + SCRD_POLICY_TOUCHID_ENROLLMENT.len() + 1 + 1 + 8 <= SCRD_MAX_LOGICAL +); + +pub(crate) struct ScrdCommand { + request: u8, + payload: [u8; SCRD_MAX_LOGICAL], + payload_len: usize, + name: &'static CStr, +} + +impl ScrdCommand { + pub(crate) fn request(&self) -> u8 { + self.request + } + pub(crate) fn payload(&self) -> &[u8] { + &self.payload[..self.payload_len] + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +fn scrd_header(out: &mut [u8; SCRD_MAX_LOGICAL], command: u8, byte5: u8, version: u8) { + out[0..4].copy_from_slice(&SCRD_MAGIC); + out[4] = command; + out[5] = byte5; + out[6] = 0; + out[7] = version; +} + +pub(crate) fn scrd_initialize() -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_INITIALIZE, SCRD_INIT_LOG_LEVEL, 0); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 8, + name: c"SCRD_INITIALIZE", + } +} + +pub(crate) fn scrd_context_create_tracked(session_uid: i32) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_CONTEXT_CREATE_TRACKED, 0, 1); + payload[8..12].copy_from_slice(&session_uid.to_le_bytes()); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 12, + name: c"SCRD_CONTEXT_CREATE", + } +} + +pub(crate) fn scrd_context_externalize(handle: &[u8; SCRD_ACM_HANDLE_LEN]) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_CONTEXT_EXTERNALIZE, 0, 1); + payload[8..8 + SCRD_ACM_HANDLE_LEN].copy_from_slice(handle); + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: 8 + SCRD_ACM_HANDLE_LEN, + name: c"SCRD_EXTERNALIZE", + } +} + +pub(crate) fn scrd_verify_touchid_enrollment(handle: &[u8; SCRD_ACM_HANDLE_LEN]) -> ScrdCommand { + let mut payload = [0u8; SCRD_MAX_LOGICAL]; + scrd_header(&mut payload, SCRD_CMD_VERIFY_POLICY, 0, 1); + let mut n = 8; + payload[n..n + SCRD_ACM_HANDLE_LEN].copy_from_slice(handle); + n += SCRD_ACM_HANDLE_LEN; + payload[n..n + SCRD_POLICY_TOUCHID_ENROLLMENT.len()] + .copy_from_slice(SCRD_POLICY_TOUCHID_ENROLLMENT); + n += SCRD_POLICY_TOUCHID_ENROLLMENT.len(); + // then NUL, preflight(1), u32 flags, u32 param count -- all zero + n += 1 + 1 + 4 + 4; + ScrdCommand { + request: SCRD_REQUEST_COMMAND, + payload, + payload_len: n, + name: c"SCRD_VERIFY_POLICY", + } +} + +pub(crate) struct ScrdReply { + pub(crate) request: u8, + pub(crate) response_size: u16, + pub(crate) status: i32, +} + +pub(crate) fn decode_scrd_reply(msg: &Message) -> ScrdReply { + let b = msg.msg0.to_le_bytes(); + ScrdReply { + request: b[1], + response_size: u16::from_le_bytes([b[2], b[3]]), + status: i32::from_le_bytes([b[4], b[5], b[6], b[7]]), + } +} + +pub(crate) fn encode_scrd(request: u8, len: usize) -> Message { + let len = (len & 0xffff) as u16; + let len = len.to_le_bytes(); + Message { + msg0: u64::from_le_bytes([EP_SCRD, request, len[0], len[1], 0, 0, 0, 0]), + msg1: 0, + } +} + +static_assert!(EP_SCRD == 0x0a); +static_assert!(SCRD_CMD_INITIALIZE == 0x0a); +static_assert!(SCRD_CMD_CONTEXT_CREATE_TRACKED == 0x24); +static_assert!(SCRD_CMD_CONTEXT_EXTERNALIZE == 0x13); +static_assert!(SCRD_CMD_VERIFY_POLICY == 0x03); diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs new file mode 100644 index 00000000000000..6b6393dc28c043 --- /dev/null +++ b/drivers/soc/apple/sensor.rs @@ -0,0 +1,736 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! The fingerprint sensor over SPI. + +use kernel::prelude::*; +use kernel::time::delay::fsleep; +use kernel::time::Delta; + +extern "C" { + fn sep_sensor_register() -> c_int; + fn sep_sensor_unregister(); + fn sep_sensor_bound() -> c_int; + fn sep_sensor_power_line() -> c_int; + fn sep_sensor_cs_timing_mode() -> c_int; + fn sep_sensor_power_cycle() -> c_int; + fn sep_sensor_power_source() -> c_int; + fn sep_sensor_power(on: c_int) -> c_int; + fn sep_sensor_xfer(tx: *const c_void, rx: *mut c_void, len: usize) -> c_int; + fn sep_sensor_xfer_tx(tx: *const c_void, len: usize) -> c_int; + fn sep_sensor_xfer2(tx: *const c_void, tx_len: usize, rx: *mut c_void, rx_len: usize) -> c_int; + fn sep_sensor_irq_setup() -> c_int; + fn sep_sensor_irq_available() -> c_int; + fn sep_sensor_irq_arm(); + fn sep_sensor_irq_wait(timeout_ms: c_uint) -> c_int; + fn sep_sensor_firmware_name() -> *const c_char; +} + +/// The calibration blob's name from the board description, or `None` when the +/// sensor node has no `firmware-name` property (the caller then uses a default). +pub(crate) fn firmware_name() -> Option<&'static CStr> { + // SAFETY: the shim returns NULL or a pointer to a NUL-terminated string + // owned by the device property and valid for the device's lifetime. + let ptr = unsafe { sep_sensor_firmware_name() }; + if ptr.is_null() { + return None; + } + // SAFETY: a non-NULL return is a valid NUL-terminated C string. + Some(unsafe { CStr::from_char_ptr(ptr) }) +} + +// Spi2. +pub(crate) const CONTROLLER_BASE: u64 = 0x3_9b10_8000; +pub(crate) const CHIP_SELECT: u32 = 0; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CsTiming { + Software, + HookBypassed, + Hardware, +} + +impl CsTiming { + fn from_wire(v: c_int) -> CsTiming { + match v { + 2 => CsTiming::Hardware, + 1 => CsTiming::HookBypassed, + _ => CsTiming::Software, + } + } + + pub(crate) fn name(self) -> &'static CStr { + match self { + CsTiming::Hardware => c"HARDWARE — programmed into the controller, which is what this sensor needs", + CsTiming::HookBypassed => { + c"emulated — the controller has a set_cs_timing hook but a GPIO chip select bypasses it" + } + CsTiming::Software => c"emulated in software — no set_cs_timing hook, and the sensor does not accept this", + } + } + + pub(crate) fn is_hardware(self) -> bool { + matches!(self, CsTiming::Hardware) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum PowerSource { + None, + NodeProperty, + ChipLine, +} + +impl PowerSource { + fn from_wire(v: c_int) -> PowerSource { + match v { + 1 => PowerSource::NodeProperty, + 2 => PowerSource::ChipLine, + _ => PowerSource::None, + } + } + + pub(crate) fn name(self) -> &'static CStr { + match self { + PowerSource::None => c"none — no power line was taken", + PowerSource::NodeProperty => c"the sensor node's own gpios property", + PowerSource::ChipLine => c"/soc/pinctrl@39b028000 line 122, taken by device-tree node", + } + } +} + +pub(crate) const POWER_ON_READ_DELAYS_MS: [u32; 4] = [0, 3, 10, 50]; + +pub(crate) const STATUS_IDENTIFIER: usize = 12; +// Known Mesa sensor revisions: 0x3352 through M4, 0x335e on newer parts. +pub(crate) const KNOWN_IDENTIFIERS: [u16; 2] = [0x3352, 0x335e]; +static_assert!(STATUS_IDENTIFIER + 2 <= STATUS_LEN); + +const CMD_LEN: usize = 7; +const CMD_GET_STATUS: [u8; CMD_LEN] = [0x80, 0x10, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_IDLE: [u8; CMD_LEN] = [0x80, 0x30, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_START_CAPTURE: [u8; CMD_LEN] = [0x80, 0x40, 0x00, 0x07, 0x00, 0x00, 0x00]; +// Precedes the patch blob. +const CMD_SETUP_PATCH_ENABLE: [u8; CMD_LEN] = [0x80, 0x60, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_GET_SENSOR_SERIAL: [u8; CMD_LEN] = [0x80, 0x70, 0x00, 0x07, 0x00, 0x00, 0x00]; + +const READ_CMD_LEN: usize = 11; +const CMD_READ_PREFIX: [u8; CMD_LEN] = [0x80, 0x13, 0x00, 0x0b, 0x00, 0x00, 0x00]; + +const STATUS_XFER_LEN: usize = 23; +const STATUS_AT: usize = 7; +pub(crate) const STATUS_LEN: usize = 16; +static_assert!(STATUS_AT + STATUS_LEN == STATUS_XFER_LEN); + +// Offsets index the 16-byte status, not the 23-byte transfer. +const STATUS_STATE: usize = STATUS_AT + 7; +const STATUS_COUNT: usize = STATUS_AT + 12; + +static_assert!(STATUS_STATE == 14); +static_assert!(STATUS_COUNT == 19); +static_assert!(STATUS_COUNT + 4 == STATUS_XFER_LEN); +static_assert!(STATUS_STATE >= STATUS_AT && STATUS_STATE < STATUS_AT + STATUS_LEN); +static_assert!(STATUS_COUNT >= STATUS_AT && STATUS_COUNT + 4 <= STATUS_AT + STATUS_LEN); + +pub(crate) const STATE_DATA_READY: u8 = 7; +pub(crate) const STATE_READING: u8 = 19; +pub(crate) const STATE_IDLE: u8 = 0; +pub(crate) const STATE_ARMED: u8 = 17; +static_assert!(STATE_ARMED != STATE_READING); +static_assert!(STATE_ARMED != STATE_DATA_READY); +static_assert!(STATE_ARMED != STATE_IDLE); +pub(crate) const STATE_NEEDS_PATCH: u8 = 9; + +// status[8], not the state byte status[7]. +pub(crate) const STATUS_PATCH_ACK: usize = 8; +pub(crate) const PATCH_ACCEPTED: u8 = 0x29; +static_assert!(STATUS_PATCH_ACK != 7); +static_assert!(STATUS_PATCH_ACK < STATUS_LEN); + +pub(crate) const MAX_CAPTURE: usize = 0x10000; + +const CRC_LEN: usize = 2; + +const FRAME_HEADER: usize = 7; +const FRAME_LEN_AT: usize = 3; +static_assert!(FRAME_LEN_AT + 2 <= FRAME_HEADER); + +const TYPE_SESSION: u8 = 0x72; +const TYPE_SEQUENCE: u8 = 0x73; +const TYPE_ENCRYPTED: u8 = 0x55; + +pub(crate) const SESSION_SHARE_LEN: usize = 40; +pub(crate) const CHALLENGE_LEN: usize = 64; + +const MAX_FRAME_PAYLOAD: usize = CHALLENGE_LEN; +static_assert!(SESSION_SHARE_LEN <= MAX_FRAME_PAYLOAD); + +pub(crate) const SESSION_REPLY_LEN: usize = 0x33; +pub(crate) const SESSION_REPLY_AT: usize = 9; +pub(crate) const CHALLENGE_REPLY_LEN: usize = 0x4b; +pub(crate) const CHALLENGE_REPLY_AT: usize = 9; + +static_assert!(SESSION_REPLY_AT + SESSION_SHARE_LEN <= SESSION_REPLY_LEN); +static_assert!(CHALLENGE_REPLY_AT + CHALLENGE_LEN <= CHALLENGE_REPLY_LEN); + +fn check(rc: c_int) -> Result<()> { + if rc < 0 { + Err(Error::from_errno(rc)) + } else { + Ok(()) + } +} + +pub(crate) fn register_driver() -> Result<()> { + // SAFETY: takes no arguments and is idempotent. + check(unsafe { sep_sensor_register() }) +} + +pub(crate) fn unregister_driver() { + // SAFETY: idempotent, and safe when nothing was ever registered. + unsafe { sep_sensor_unregister() } +} + +pub(crate) fn is_bound() -> bool { + // SAFETY: reads one pointer for nullness. + unsafe { sep_sensor_bound() != 0 } +} + +pub(crate) fn power_line() -> Option { + // SAFETY: no arguments; returns a negative errno when unbound. + let n = unsafe { sep_sensor_power_line() }; + if n < 0 { + None + } else { + Some(n) + } +} + +pub(crate) fn power_cycle() -> bool { + // SAFETY: the shim holds the descriptor and applies its delays. + unsafe { sep_sensor_power_cycle() == 0 } +} + +pub(crate) fn cs_timing() -> CsTiming { + // SAFETY: reads one int. + CsTiming::from_wire(unsafe { sep_sensor_cs_timing_mode() }) +} + +pub(crate) fn power_source() -> PowerSource { + // SAFETY: reads one int. + PowerSource::from_wire(unsafe { sep_sensor_power_source() }) +} + +pub(crate) fn power(on: bool) -> bool { + // SAFETY: the shim holds the descriptor and applies its delays. + unsafe { sep_sensor_power(if on { 1 } else { 0 }) == 0 } +} + +/// Sets up the data-ready interrupt once, while the sensor is idle. Returns +/// true when an interrupt is available for interrupt-driven capture. +pub(crate) fn irq_setup() -> bool { + // SAFETY: no arguments; idempotent, falls back to -errno when unavailable. + unsafe { sep_sensor_irq_setup() == 0 } +} + +pub(crate) fn irq_available() -> bool { + // SAFETY: reads one int. + unsafe { sep_sensor_irq_available() != 0 } +} + +/// Clears any stale data-ready signal before a capture. +pub(crate) fn irq_arm() { + // SAFETY: no arguments; a no-op when no IRQ is configured. + unsafe { sep_sensor_irq_arm() } +} + +/// Waits up to `timeout_ms` for the data-ready line to assert. Returns true if +/// it fired, false on timeout or when no interrupt is configured. +pub(crate) fn irq_wait(timeout_ms: c_uint) -> bool { + // SAFETY: no pointers; the shim owns the completion and the IRQ line. + unsafe { sep_sensor_irq_wait(timeout_ms) == 0 } +} + +fn command(cmd: &[u8; CMD_LEN]) -> Result<()> { + let mut rx = [0u8; CMD_LEN]; + // SAFETY: both buffers are `CMD_LEN` bytes and live across the call. + check(unsafe { sep_sensor_xfer(cmd.as_ptr().cast(), rx.as_mut_ptr().cast(), CMD_LEN) }) +} + +pub(crate) fn idle() -> Result<()> { + command(&CMD_IDLE) +} + +pub(crate) fn start_capture() -> Result<()> { + command(&CMD_START_CAPTURE) +} + +pub(crate) fn setup_patch_enable() -> Result<()> { + command(&CMD_SETUP_PATCH_ENABLE) +} + +fn send_framed(frame_type: u8, payload: &[u8]) -> Result<()> { + let total = FRAME_HEADER + payload.len(); + if payload.len() > MAX_FRAME_PAYLOAD || total > u16::MAX as usize { + return Err(EINVAL); + } + + let mut frame = [0u8; FRAME_HEADER + MAX_FRAME_PAYLOAD]; + frame[0] = 0x80; + frame[1] = frame_type; + frame[FRAME_LEN_AT..FRAME_LEN_AT + 2].copy_from_slice(&(total as u16).to_le_bytes()); + frame[FRAME_HEADER..total].copy_from_slice(payload); + + // SAFETY: `frame` is at least `total` bytes and lives across the call; the + // shim only reads from it and there is no receive buffer. + check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), total) }) +} + +const ENCRYPTED_OVERHEAD: usize = FRAME_HEADER + CRC_LEN; +static_assert!(ENCRYPTED_OVERHEAD == 9); +const ENCRYPTED_TRANSFER_MAX: usize = 0x12b; +static_assert!(ENCRYPTED_TRANSFER_MAX < 0x12c); + +pub(crate) struct Geometry { + declared: usize, + transfer: usize, +} + +impl Geometry { + // 0x5d. + pub(crate) const COVERAGE: Geometry = Geometry { + declared: 0x40, + transfer: 0x40, + }; + // 0x5c. + pub(crate) const OPERATION: Geometry = Geometry { + declared: 0x40, + transfer: 0x40, + }; + pub(crate) const MODULE_CHALLENGE: Geometry = Geometry { + declared: 0x49, + transfer: 0x49, + }; + + // 0x6a, transmitted zero-padded past declared. + pub(crate) const TRANSPARENT: Geometry = Geometry { + declared: 0x4d, + transfer: 0x12b, + }; + + pub(crate) const fn payload_capacity(&self) -> usize { + self.declared - ENCRYPTED_OVERHEAD + } + + pub(crate) fn declared(&self) -> usize { + self.declared + } +} + +static_assert!(Geometry::MODULE_CHALLENGE.declared < 0x12c); +static_assert!(Geometry::MODULE_CHALLENGE.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::MODULE_CHALLENGE.transfer >= Geometry::MODULE_CHALLENGE.declared); +static_assert!(Geometry::MODULE_CHALLENGE.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::MODULE_CHALLENGE.payload_capacity() == 0x40); +static_assert!(Geometry::COVERAGE.declared < 0x12c); +static_assert!(Geometry::OPERATION.declared < 0x12c); +static_assert!(Geometry::TRANSPARENT.declared < 0x12c); +static_assert!(Geometry::COVERAGE.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::OPERATION.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::TRANSPARENT.declared > ENCRYPTED_OVERHEAD); +static_assert!(Geometry::COVERAGE.transfer >= Geometry::COVERAGE.declared); +static_assert!(Geometry::OPERATION.transfer >= Geometry::OPERATION.declared); +static_assert!(Geometry::TRANSPARENT.transfer >= Geometry::TRANSPARENT.declared); +static_assert!(Geometry::COVERAGE.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::OPERATION.transfer <= ENCRYPTED_TRANSFER_MAX); +static_assert!(Geometry::TRANSPARENT.transfer <= ENCRYPTED_TRANSFER_MAX); + +fn crc16_ansi(data: &[u8]) -> u16 { + let mut crc: u16 = 0; + for &byte in data { + crc ^= u16::from(byte); + for _ in 0..8 { + if crc & 1 != 0 { + crc = (crc >> 1) ^ 0xA001; + } else { + crc >>= 1; + } + } + } + crc +} + +pub(crate) struct Params(KVec); + +impl Params { + pub(crate) fn new(blob: KVec) -> Params { + Params(blob) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn len(&self) -> usize { + self.0.len() + } +} + +impl Drop for Params { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) enum ParamsError { + Empty, + TooLong(usize, usize), + Transfer(Error), +} + +pub(crate) fn send_encrypted_parameters( + params: &Params, + geom: &Geometry, +) -> core::result::Result<(), ParamsError> { + if params.0.is_empty() { + return Err(ParamsError::Empty); + } + let capacity = geom.payload_capacity(); + if params.0.len() > capacity { + return Err(ParamsError::TooLong(params.0.len(), capacity)); + } + + let declared = geom.declared; + let transfer = geom.transfer; + let mut frame = [0u8; ENCRYPTED_TRANSFER_MAX]; + + frame[0] = 0x80; + frame[1] = TYPE_ENCRYPTED; + // Declared size, not the transferred count; the tail past it is zero padding. + frame[FRAME_LEN_AT..FRAME_LEN_AT + 2].copy_from_slice(&(declared as u16).to_le_bytes()); + frame[FRAME_HEADER..FRAME_HEADER + params.0.len()].copy_from_slice(¶ms.0); + + let crc_at = declared - CRC_LEN; + let crc = crc16_ansi(&frame[..crc_at]); + frame[crc_at..declared].copy_from_slice(&crc.to_le_bytes()); + + // SAFETY: `frame` is `ENCRYPTED_TRANSFER_MAX` bytes, `transfer` is no + // larger (asserted for every `Geometry`), and it lives across the call; the + // shim only reads from it and there is no receive buffer. + check(unsafe { sep_sensor_xfer_tx(frame.as_ptr().cast(), transfer) }) + .map_err(ParamsError::Transfer) +} + +pub(crate) const MODULE_REPLY_LENS: [usize; 2] = [0x4b, 0x4f]; +static_assert!(MODULE_REPLY_LENS[0] <= MAX_REPLY); +static_assert!(MODULE_REPLY_LENS[1] <= MAX_REPLY); +static_assert!(MODULE_REPLY_LENS[0] != MODULE_REPLY_LENS[1]); + +const MODULE_REPLY_ADJUST: usize = 11; +const MODULE_REPLY_AT: usize = 9; +static_assert!(MODULE_REPLY_AT == SESSION_REPLY_AT); +static_assert!(MODULE_REPLY_AT == CHALLENGE_REPLY_AT); +static_assert!(MODULE_REPLY_AT == SENSOR_SERIAL_AT); +static_assert!(MODULE_REPLY_ADJUST == MODULE_REPLY_AT + CRC_LEN); +static_assert!(MODULE_REPLY_LENS[0] > MODULE_REPLY_ADJUST); + +pub(crate) fn send_module_challenge(challenge: &Params) -> core::result::Result<(), ParamsError> { + send_encrypted_parameters(challenge, &Geometry::MODULE_CHALLENGE) +} + +pub(crate) fn read_module_reply() -> Result> { + let count = poll_until_ready_any(&MODULE_REPLY_LENS)?; + + let mut buf = [0xffu8; MAX_REPLY]; + read_reply_ready(count, &mut buf)?; + + // Relayed unparsed to the enclave, op 0x32. + let len = count - MODULE_REPLY_ADJUST; + let mut out = KVec::new(); + out.extend_from_slice(&buf[MODULE_REPLY_AT..MODULE_REPLY_AT + len], GFP_KERNEL)?; + Ok(out) +} + +fn poll_until_ready_any(accepted: &[usize]) -> Result { + for _ in 0..READ_POLL_ATTEMPTS { + let st = status()?; + if let Offset12::Available(count) = st.offset12() { + let count = count as usize; + if accepted.contains(&count) { + return Ok(count); + } + } + fsleep(Delta::from_millis(READ_POLL_MS)); + } + Err(ETIMEDOUT) +} + +pub(crate) fn send_session_share(share: &[u8; SESSION_SHARE_LEN]) -> Result<()> { + send_framed(TYPE_SESSION, share) +} + +pub(crate) fn send_challenge(challenge: &[u8; CHALLENGE_LEN]) -> Result<()> { + send_framed(TYPE_SEQUENCE, challenge) +} + +static_assert!(FRAME_HEADER + SESSION_SHARE_LEN == 0x2f); +static_assert!(FRAME_HEADER + CHALLENGE_LEN == 0x47); + +fn read_framed(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + let mut buf = [0xffu8; MAX_REPLY]; + read_reply(total, &mut buf)?; + if at + out.len() > total { + return Err(EINVAL); + } + out.copy_from_slice(&buf[at..at + out.len()]); + Ok(()) +} + +fn poll_until_ready(expected: usize) -> Result<()> { + for _ in 0..READ_POLL_ATTEMPTS { + let st = status()?; + if let Offset12::Available(count) = st.offset12() { + if count as usize == expected { + return Ok(()); + } + } + fsleep(Delta::from_millis(READ_POLL_MS)); + } + Err(ETIMEDOUT) +} + +const READ_POLL_MS: i64 = 5; +const READ_POLL_ATTEMPTS: usize = 200; + +fn read_reply(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { + poll_until_ready(total)?; + read_reply_ready(total, buf) +} + +fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { + if total > MAX_REPLY { + return Err(EINVAL); + } + + let mut cmd = [0u8; READ_CMD_LEN]; + cmd[..CMD_LEN].copy_from_slice(&CMD_READ_PREFIX); + cmd[CMD_LEN..].copy_from_slice(&(total as u32).to_le_bytes()); + + // SAFETY: `cmd` is `READ_CMD_LEN` bytes, `buf` is `MAX_REPLY` and `total` + // is bounded by it; both live across the call and the shim writes only into + // the receive buffer. + check(unsafe { + sep_sensor_xfer2( + cmd.as_ptr().cast(), + READ_CMD_LEN, + buf.as_mut_ptr().cast(), + total, + ) + }) +} + +fn read_framed_verified(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + if total < CRC_LEN || at + out.len() > total - CRC_LEN { + return Err(EINVAL); + } + let mut buf = [0xffu8; MAX_REPLY]; + read_reply(total, &mut buf)?; + + let split = total - CRC_LEN; + let expected = u16::from_le_bytes([buf[split], buf[split + 1]]); + let computed = crc16_ansi(&buf[..split]); + if expected != computed { + return Err(EIO); + } + + out.copy_from_slice(&buf[at..at + out.len()]); + Ok(()) +} + +const MAX_REPLY: usize = 0x4f; +static_assert!(SESSION_REPLY_LEN <= MAX_REPLY); +static_assert!(CHALLENGE_REPLY_LEN <= MAX_REPLY); +static_assert!(SENSOR_SERIAL_REPLY_LEN <= MAX_REPLY); + +pub(crate) const SENSOR_SERIAL_REPLY_LEN: usize = 0x1b; +pub(crate) const SENSOR_SERIAL_LEN: usize = 16; +pub(crate) const SENSOR_SERIAL_AT: usize = 9; +static_assert!(SENSOR_SERIAL_AT == SESSION_REPLY_AT); +static_assert!(SENSOR_SERIAL_AT + SENSOR_SERIAL_LEN <= SENSOR_SERIAL_REPLY_LEN - CRC_LEN); + +pub(crate) struct SensorSerial([u8; SENSOR_SERIAL_LEN]); + +impl SensorSerial { + pub(crate) fn bytes(&self) -> &[u8; SENSOR_SERIAL_LEN] { + &self.0 + } +} + +pub(crate) fn read_sensor_serial() -> Result { + command(&CMD_GET_SENSOR_SERIAL)?; + let mut serial = [0u8; SENSOR_SERIAL_LEN]; + read_framed_verified(SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; + Ok(SensorSerial(serial)) +} + +pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { + let mut share = [0u8; SESSION_SHARE_LEN]; + read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; + Ok(share) +} + +pub(crate) fn read_challenge_reply() -> Result<[u8; CHALLENGE_LEN]> { + let mut reply = [0u8; CHALLENGE_LEN]; + read_framed(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; + Ok(reply) +} + +pub(crate) fn send_patch(blob: &[u8]) -> Result<()> { + if blob.is_empty() { + return Err(EINVAL); + } + // SAFETY: `blob` is a live slice for the duration of the call and the shim + // only reads from it; there is no receive buffer to write into. + check(unsafe { sep_sensor_xfer_tx(blob.as_ptr().cast(), blob.len()) }) +} + +pub(crate) struct Status { + pub(crate) state: u8, + offset12: u32, + pub(crate) raw: [u8; STATUS_LEN], +} + +pub(crate) enum Offset12 { + Identifier(u16), + Available(u32), + Undefined, +} + +pub(crate) struct Identifier(u16); + +impl Identifier { + pub(crate) fn value(&self) -> u16 { + self.0 + } +} + +impl Status { + pub(crate) fn offset12(&self) -> Offset12 { + match self.state { + STATE_IDLE => Offset12::Identifier(u16::from_le_bytes([ + self.raw[STATUS_IDENTIFIER], + self.raw[STATUS_IDENTIFIER + 1], + ])), + STATE_DATA_READY => Offset12::Available(self.offset12), + _ => Offset12::Undefined, + } + } + + pub(crate) fn identifier_to_register(&self) -> Option { + match self.offset12() { + Offset12::Identifier(0) => None, + Offset12::Identifier(value) => Some(Identifier(value)), + Offset12::Available(_) | Offset12::Undefined => None, + } + } + + pub(crate) fn patch_ack(&self) -> u8 { + self.raw[STATUS_PATCH_ACK] + } + + pub(crate) fn is_silent(&self) -> bool { + self.raw.iter().all(|b| *b == 0) + } +} + +pub(crate) fn status() -> Result { + let mut tx = [0xffu8; STATUS_XFER_LEN]; + tx[..CMD_LEN].copy_from_slice(&CMD_GET_STATUS); + let mut rx = [0u8; STATUS_XFER_LEN]; + + // SAFETY: both buffers are `STATUS_XFER_LEN` bytes and live across the call. + check(unsafe { sep_sensor_xfer(tx.as_ptr().cast(), rx.as_mut_ptr().cast(), STATUS_XFER_LEN) })?; + + let mut raw = [0u8; STATUS_LEN]; + raw.copy_from_slice(&rx[STATUS_AT..STATUS_AT + STATUS_LEN]); + + Ok(Status { + state: rx[STATUS_STATE], + offset12: u32::from_le_bytes([ + rx[STATUS_COUNT], + rx[STATUS_COUNT + 1], + rx[STATUS_COUNT + 2], + rx[STATUS_COUNT + 3], + ]), + raw, + }) +} + +pub(crate) struct Capture(KVec); + +impl Capture { + pub(crate) fn bytes(&self) -> &[u8] { + &self.0 + } +} + +impl Drop for Capture { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +pub(crate) enum CaptureError { + Length(u32), + Bus(Error), + NoMemory, + Checksum { advertised: u16, computed: u16 }, +} + +pub(crate) fn read_capture(length: u32) -> core::result::Result { + let len = length as usize; + if length == 0 || len > MAX_CAPTURE || len <= CRC_LEN { + return Err(CaptureError::Length(length)); + } + + let mut cmd = [0u8; READ_CMD_LEN]; + cmd[..CMD_LEN].copy_from_slice(&CMD_READ_PREFIX); + cmd[CMD_LEN..].copy_from_slice(&length.to_le_bytes()); + + let mut buf = KVec::with_capacity(len, GFP_KERNEL).map_err(|_| CaptureError::NoMemory)?; + buf.resize(len, 0xff, GFP_KERNEL) + .map_err(|_| CaptureError::NoMemory)?; + + // SAFETY: `cmd` is `READ_CMD_LEN` bytes and `buf` is `len` bytes; both live + // across the call, and the shim writes only into the receive buffer. + let rc = unsafe { + sep_sensor_xfer2( + cmd.as_ptr().cast(), + READ_CMD_LEN, + buf.as_mut_ptr().cast(), + len, + ) + }; + if rc < 0 { + return Err(CaptureError::Bus(Error::from_errno(rc))); + } + + let split = len - CRC_LEN; + let advertised = u16::from_le_bytes([buf[split], buf[split + 1]]); + // CRC-16/ANSI, not CCITT-FALSE. + let computed = crc16_ansi(&buf[..split]); + if advertised != computed { + return Err(CaptureError::Checksum { + advertised, + computed, + }); + } + + Ok(Capture(buf)) +} diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c new file mode 100644 index 00000000000000..aff7da78fe49e3 --- /dev/null +++ b/drivers/soc/apple/sensor_shim.c @@ -0,0 +1,572 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* Fingerprint sensor SPI shim: moves bytes over the bus and toggles power. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +/* + * 8 MHz, SPI mode 2 (CPOL=1 CPHA=0), 8-bit words. Mode 3 returns sixteen zero + * bytes, else identical. Must agree with the node's spi-cpol / absent spi-cpha, + * which is what spi_setup() applies. + */ +#define SEP_SENSOR_HZ 8000000 +#define SEP_SENSOR_BITS 8 +static unsigned int sep_sensor_mode = SPI_MODE_2; +module_param_named(sensor_spi_mode, sep_sensor_mode, uint, 0444); +MODULE_PARM_DESC(sensor_spi_mode, + "SPI mode for the sensor: 2 (CPOL=1 CPHA=0, the default and the only mode this sensor answers under) or 3 (CPOL=1 CPHA=1, reads all-zero). For comparison only."); +#define SEP_SENSOR_MODE sep_sensor_mode + +/* Chip-select setup and hold, in ns. */ +#define SEP_SENSOR_CS_NS 20 + +#define SEP_SENSOR_OFF_DELAY_MS 10 +#define SEP_SENSOR_ON_DELAY_MS 7 + +static struct spi_device *sep_spi; +static struct gpio_desc *sep_power; +static bool sep_registered; + +/* + * Power line resolved by DT node path, not gpiochip index: registration order + * shifts, so an index would silently pick the wrong chip. + */ +#define SEP_SENSOR_GPIO_NODE "/soc/pinctrl@39b028000" +#define SEP_SENSOR_GPIO_LINE 122 + +#define SEP_POWER_NONE 0 +#define SEP_POWER_NODE_PROPERTY 1 +#define SEP_POWER_CHIP_LINE 2 + +static int sep_power_source; + +/* + * Data-ready line for interrupt-driven capture. On j414s it is the pin adjacent + * to the power line (122) on the same controller; other boards describe it as + * the SPI node's interrupt, which the SPI core resolves into spi->irq. + */ +#define SEP_SENSOR_DRDY_LINE 121 +static struct gpio_desc *sep_drdy; /* set only when we own the gpiochip line (j414s) */ +static int sep_drdy_irq = -1; +static DECLARE_COMPLETION(sep_drdy_done); + +/* + * Takes the power line as an output driven low: the power cycle begins with an + * off phase, so the line must be actively driven off, not merely read as low. + */ +static void sep_acquire_power(struct spi_device *spi) +{ + struct device_node *np; + struct gpio_device *gdev; + struct gpio_chip *gc; + + sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); + if (!IS_ERR(sep_power)) { + sep_power_source = SEP_POWER_NODE_PROPERTY; + dev_info(&spi->dev, + "sep sensor: power line from the device node, driven low\n"); + return; + } + sep_power = NULL; + + /* + * The fallback below is the j414s power line; a sibling board's sensor sits + * on a different pin, so only j414s may use it. Every other machine must + * describe the power GPIO in its device node (the gpiod_get_index path). + */ + if (!of_machine_is_compatible("apple,j414s")) { + dev_warn(&spi->dev, + "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); + return; + } + + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); + if (!np) { + dev_warn(&spi->dev, + "sep sensor: no DT node at %s, no power line\n", + SEP_SENSOR_GPIO_NODE); + return; + } + + gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); + of_node_put(np); + if (!gdev) { + dev_warn(&spi->dev, + "sep sensor: %s has no registered GPIO device\n", + SEP_SENSOR_GPIO_NODE); + return; + } + + gc = gpio_device_get_chip(gdev); + if (gc) + sep_power = gpiochip_request_own_desc(gc, + SEP_SENSOR_GPIO_LINE, + "apple-mesa-power", + GPIO_LOOKUP_FLAGS_DEFAULT, + GPIOD_OUT_LOW); + if (!gc || IS_ERR(sep_power)) { + sep_power = NULL; + dev_warn(&spi->dev, + "sep sensor: could not take line %d on %s (%s)\n", + SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, + gpio_device_get_label(gdev)); + gpio_device_put(gdev); + return; + } + + sep_power_source = SEP_POWER_CHIP_LINE; + dev_info(&spi->dev, + "sep sensor: power line %s line %d (%s), driven low\n", + SEP_SENSOR_GPIO_NODE, SEP_SENSOR_GPIO_LINE, + gpio_device_get_label(gdev)); + gpio_device_put(gdev); +} + +static void sep_release_power(void) +{ + if (!sep_power) + return; + gpiod_set_value_cansleep(sep_power, 0); + if (sep_power_source == SEP_POWER_CHIP_LINE) + gpiochip_free_own_desc(sep_power); + else + gpiod_put(sep_power); + sep_power = NULL; + sep_power_source = SEP_POWER_NONE; +} + +#define SEP_CS_TIMING_SOFTWARE 0 +#define SEP_CS_TIMING_HOOK_BYPASSED 1 +#define SEP_CS_TIMING_HARDWARE 2 + +static int sep_cs_timing_mode; + +/* + * Programs the 20 ns chip-select setup/hold and applies mode/speed via + * spi_setup(), which reaches the controller's set_cs_timing hook. + * + * This sensor needs hardware timing. The hook runs only for a native chip + * select; with a GPIO chip select the core silently emulates the delays in + * software, which this sensor rejects but which looks like success. + */ +static int sep_apply_cs_timing(struct spi_device *spi) +{ + struct spi_controller *ctlr = spi->controller; + int rc; + + if (!ctlr->set_cs_timing) + sep_cs_timing_mode = SEP_CS_TIMING_SOFTWARE; + else if (spi_get_csgpiod(spi, 0)) + sep_cs_timing_mode = SEP_CS_TIMING_HOOK_BYPASSED; + else + sep_cs_timing_mode = SEP_CS_TIMING_HARDWARE; + + /* + * Hold the bus lock: these fields are read when a transfer asserts chip + * select. spi_setup() takes the controller io_mutex, not the bus lock, + * so no deadlock. + */ + rc = spi_bus_lock(ctlr); + if (rc) + return rc; + + spi->mode = SEP_SENSOR_MODE; + spi->bits_per_word = SEP_SENSOR_BITS; + spi->max_speed_hz = SEP_SENSOR_HZ; + spi->cs_setup.value = SEP_SENSOR_CS_NS; + spi->cs_setup.unit = SPI_DELAY_UNIT_NSECS; + spi->cs_hold.value = SEP_SENSOR_CS_NS; + spi->cs_hold.unit = SPI_DELAY_UNIT_NSECS; + + rc = spi_setup(spi); + + spi_bus_unlock(ctlr); + + if (rc) + return rc; + + switch (sep_cs_timing_mode) { + case SEP_CS_TIMING_HARDWARE: + dev_info(&spi->dev, + "sep sensor: CS timing %u ns programmed in hardware\n", + SEP_SENSOR_CS_NS); + break; + case SEP_CS_TIMING_HOOK_BYPASSED: + dev_warn(&spi->dev, + "sep sensor: GPIO chip select, CS timing emulated in software (sensor needs hardware timing)\n"); + break; + default: + dev_warn(&spi->dev, + "sep sensor: no set_cs_timing hook, %u ns emulated in software (sensor needs hardware timing)\n", + SEP_SENSOR_CS_NS); + break; + } + return 0; +} + +static int sep_sensor_probe(struct spi_device *spi) +{ + int rc; + + rc = sep_apply_cs_timing(spi); + if (rc) + return rc; + + sep_spi = spi; + sep_acquire_power(spi); + /* Read the mode back: a mode that did not take must not be logged as + * the one requested. */ + dev_info(&spi->dev, + "sep sensor: bound, %u Hz, mode %u (CPOL=%d CPHA=%d)\n", + spi->max_speed_hz, + (unsigned int)(spi->mode & (SPI_CPOL | SPI_CPHA)), + !!(spi->mode & SPI_CPOL), !!(spi->mode & SPI_CPHA)); + return 0; +} + +static void sep_sensor_remove(struct spi_device *spi) +{ + sep_sensor_irq_teardown(); + sep_release_power(); + sep_spi = NULL; +} + +/* + * Its own compatible, deliberately not spidev's: capture must stay in the + * kernel, and a spidev node would leak an image to userspace. No + * MODULE_DEVICE_TABLE, so udev cannot auto-load this module. + */ +static const struct of_device_id sep_sensor_of_match[] = { + { .compatible = "apple,mesa-fingerprint" }, + { } +}; + +/* + * Legacy ID table only to silence the SPI core's "no spi_device_id" note. No + * MODULE_DEVICE_TABLE: it quiets a message, it does not advertise a binding. + */ +static const struct spi_device_id sep_sensor_spi_ids[] = { + { "mesa-fingerprint", 0 }, + { } +}; + +static struct spi_driver sep_sensor_driver = { + .driver = { + .name = "apple-mesa", + .of_match_table = sep_sensor_of_match, + }, + .id_table = sep_sensor_spi_ids, + .probe = sep_sensor_probe, + .remove = sep_sensor_remove, +}; + +/* + * Registered before the device tree gains the sensor node, so the bind happens + * when the SPI core's notifier creates the device. + */ +int sep_sensor_register(void) +{ + int rc; + + if (sep_registered) + return 0; + rc = spi_register_driver(&sep_sensor_driver); + if (rc) + return rc; + sep_registered = true; + return 0; +} + +void sep_sensor_unregister(void) +{ + if (!sep_registered) + return; + spi_unregister_driver(&sep_sensor_driver); + sep_registered = false; +} + +int sep_sensor_bound(void) +{ + return sep_spi != NULL; +} + +/* Whether those delays reach hardware, are emulated, or are silently bypassed. */ +int sep_sensor_cs_timing_mode(void) +{ + return sep_cs_timing_mode; +} + +/* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ +int sep_sensor_power_cycle(void) +{ + if (!sep_power) + return -ENODEV; + + gpiod_set_value_cansleep(sep_power, 0); + msleep(SEP_SENSOR_OFF_DELAY_MS); + gpiod_set_value_cansleep(sep_power, 1); + msleep(SEP_SENSOR_ON_DELAY_MS); + return 0; +} + +/* How the power line was obtained: none, the node's property, or chip+line. */ +int sep_sensor_power_source(void) +{ + return sep_power_source; +} + +int sep_sensor_power_line(void) +{ + if (!sep_power) + return -ENODEV; + return desc_to_gpio(sep_power); +} + +/* + * The per-device calibration blob name, as the board description gives it in + * the sensor node's "firmware-name" property. NULL when absent, so the caller + * can fall back to a default. + */ +const char *sep_sensor_firmware_name(void) +{ + const char *name = NULL; + + if (!sep_spi) + return NULL; + if (device_property_read_string(&sep_spi->dev, "firmware-name", &name)) + return NULL; + return name; +} + +/* Powers the sensor on/off, holding the hardware settling delay. */ +int sep_sensor_power(int on) +{ + if (!sep_power) + return -ENODEV; + + gpiod_set_value_cansleep(sep_power, on ? 1 : 0); + if (on) + msleep(SEP_SENSOR_ON_DELAY_MS); + else + msleep(SEP_SENSOR_OFF_DELAY_MS); + return 0; +} + +/* + * Threaded data-ready handler. IRQF_ONESHOT keeps the line masked while this + * runs, so a level-asserted line cannot storm the CPU; it only wakes the + * capture loop, which still confirms the sensor state over SPI before reading. + */ +static irqreturn_t sep_drdy_isr(int irq, void *dev_id) +{ + complete(&sep_drdy_done); + return IRQ_HANDLED; +} + +/* + * Set up the data-ready interrupt once, while the sensor is idle. It is left + * configured and enabled for the driver's lifetime -- never toggled per capture + * -- because reconfiguring this line mid-capture is what drops the sensor's + * firmware patch. Returns 0 when an interrupt is available, -errno otherwise + * (the caller then falls back to polling). + */ +int sep_sensor_irq_setup(void) +{ + struct device_node *np; + struct gpio_device *gdev; + struct gpio_chip *gc; + int irq, rc; + + if (!sep_spi) + return -ENODEV; + if (sep_drdy_irq >= 0) + return 0; + + if (sep_spi->irq > 0) { + /* A board that describes the data-ready line as the SPI node's + * interrupt gets it resolved by the SPI core. */ + irq = sep_spi->irq; + } else if (of_machine_is_compatible("apple,j414s")) { + np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); + if (!np) + return -ENODEV; + gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); + of_node_put(np); + if (!gdev) + return -ENODEV; + gc = gpio_device_get_chip(gdev); + if (!gc) { + gpio_device_put(gdev); + return -ENODEV; + } + sep_drdy = gpiochip_request_own_desc(gc, SEP_SENSOR_DRDY_LINE, + "apple-mesa-drdy", + GPIO_LOOKUP_FLAGS_DEFAULT, + GPIOD_IN); + gpio_device_put(gdev); + if (IS_ERR(sep_drdy)) { + sep_drdy = NULL; + return -ENODEV; + } + irq = gpiod_to_irq(sep_drdy); + if (irq < 0) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + return irq; + } + } else { + return -ENODEV; + } + + init_completion(&sep_drdy_done); + /* + * Edge-triggered on the rising edge only. The line idles low, so a level + * trigger storms and both-edges fires again on the deassert -- a spurious + * wake that, mid-enrolment, spins the capture loop. The data-ready assert + * is the low->high edge; the capture loop's status read gates the frame, + * and a missed edge only costs one status poll. + */ + rc = request_threaded_irq(irq, NULL, sep_drdy_isr, + IRQF_ONESHOT | IRQF_TRIGGER_RISING, + "apple-mesa-drdy", sep_spi); + if (rc) { + if (sep_drdy) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + } + return rc; + } + sep_drdy_irq = irq; + dev_info(&sep_spi->dev, + "sep sensor: data-ready IRQ %d configured for interrupt capture\n", + irq); + return 0; +} + +void sep_sensor_irq_teardown(void) +{ + if (sep_drdy_irq >= 0) { + free_irq(sep_drdy_irq, sep_spi); + sep_drdy_irq = -1; + } + if (sep_drdy) { + gpiochip_free_own_desc(sep_drdy); + sep_drdy = NULL; + } +} + +int sep_sensor_irq_available(void) +{ + return sep_drdy_irq >= 0; +} + +/* Clear any stale signal before a capture so the next wait reflects a fresh + * data-ready edge, not a leftover from the previous frame. */ +void sep_sensor_irq_arm(void) +{ + if (sep_drdy_irq >= 0) + reinit_completion(&sep_drdy_done); +} + +/* + * Wait for the data-ready line to assert, up to timeout_ms. 0 = fired, + * -ETIMEDOUT = no signal (the caller re-polls status over SPI regardless, so a + * missed interrupt only costs one poll interval), -ENODEV = no interrupt. + */ +int sep_sensor_irq_wait(unsigned int timeout_ms) +{ + if (sep_drdy_irq < 0) + return -ENODEV; + if (wait_for_completion_timeout(&sep_drdy_done, + msecs_to_jiffies(timeout_ms)) == 0) + return -ETIMEDOUT; + return 0; +} + +/* + * One chip-select assertion, `len` clocks, full duplex. The caller supplies all + * `len` transmit bytes (trailing ones as 0xff) rather than relying on the + * controller running its transmit buffer dry. + */ +int sep_sensor_xfer(const void *tx, void *rx, size_t len) +{ + struct spi_transfer xfer = { + .tx_buf = tx, + .rx_buf = rx, + .len = len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }; + + if (!sep_spi) + return -ENODEV; + if (!len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, &xfer, 1); +} + +/* + * One chip-select assertion, transmit only, rx_buf NULL (not a discard buffer). + * A simultaneous receive changes the long transfer's pacing and the sensor + * rejects the patch blob, so the receive must be absent entirely. + */ +int sep_sensor_xfer_tx(const void *tx, size_t len) +{ + struct spi_transfer xfer = { + .tx_buf = tx, + .rx_buf = NULL, + .len = len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }; + + if (!sep_spi) + return -ENODEV; + if (!len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, &xfer, 1); +} + +/* + * Two transfers in one chip-select assertion: command out, then read in. + * Chip-select stays asserted because neither transfer sets cs_change. + */ +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +{ + struct spi_transfer xfers[2] = { + { + .tx_buf = tx, + .len = tx_len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }, + { + .rx_buf = rx, + .len = rx_len, + .speed_hz = SEP_SENSOR_HZ, + .bits_per_word = SEP_SENSOR_BITS, + }, + }; + + if (!sep_spi) + return -ENODEV; + if (!tx_len || !rx_len) + return -EINVAL; + + return spi_sync_transfer(sep_spi, xfers, 2); +} diff --git a/drivers/soc/apple/sep-bio.h b/drivers/soc/apple/sep-bio.h new file mode 100644 index 00000000000000..7ad0b814af6484 --- /dev/null +++ b/drivers/soc/apple/sep-bio.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#pragma once + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; + __u32 enroll_stages; + __u32 reserved[3]; +}; + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 reserved; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR (SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR (SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW (SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW (SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR (SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO (SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW (SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO (SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index bcceb7ed4a6c41..61468bb8acd226 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1,15 +1,44 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! Driver for the Apple SEP (Secure Enclave Processor): attaches to a running +//! SEP over the AP mailbox, drives Touch ID enrol/verify via `/dev/sep-bio`, +//! and registers a hwrng and a SEP-backed `trusted` key source. #![recursion_limit = "2048"] -//! Apple SEP driver -//! -//! Copyright (C) The Asahi Linux Contributors +#[cfg(not(CONFIG_OF_DYNAMIC))] +compile_error!("apple_sep requires CONFIG_OF_DYNAMIC: the SEP and DART nodes ship disabled and are enabled with a device-tree changeset"); + +mod bio; +mod catacomb; +mod control; +mod der; +mod dt; +mod fv; +mod hwrng; +mod image; +mod keybag; +mod profile; +mod proto; +mod refkey; +mod refkey_seal; +mod rxring; +mod sbio; +mod scrd; +mod sensor; +mod shim; +mod shmem; +mod sks; +mod store; +mod transfer; +mod trusted; +mod xarm; +mod xart_store; use kernel::{ bindings, device, dma, - module_platform_driver, + driver, new_mutex, of, platform, @@ -25,346 +54,2141 @@ use kernel::{ Atomic, Relaxed, // }, + new_condvar, Arc, + CondVar, + CondVarTimeoutResult, Mutex, // }, + time, types::ForeignOwnable, workqueue::{ self, + impl_has_delayed_work, impl_has_work, + new_delayed_work, new_work, + DelayedWork, Work, WorkItem, // }, // }; -const SHMEM_SIZE: usize = 0x30000; -const MSG_BOOT_TZ0: u64 = 0x5; -const MSG_BOOT_IMG4: u64 = 0x6; -const MSG_SET_SHMEM: u64 = 0x18; -const MSG_BOOT_TZ0_ACK1: u64 = 0x69; -const MSG_BOOT_TZ0_ACK2: u64 = 0xD2; -const MSG_BOOT_IMG4_ACK: u64 = 0x6A; -const MSG_ADVERTISE_EP: u64 = 0; -const EP_DISCOVER: u64 = 0xFD; -const EP_SHMEM: u64 = 0xFE; -const EP_BOOT: u64 = 0xFF; - -const MSG_TYPE_SHIFT: u32 = 16; -const MSG_TYPE_MASK: u64 = 0xFF; -//const MSG_PARAM_SHIFT: u32 = 24; -//const MSG_PARAM_MASK: u64 = 0xFF; - -const MSG_EP_MASK: u64 = 0xFF; -const MSG_DATA_SHIFT: u32 = 32; - -const IOVA_SHIFT: u32 = 0xC; - -type ShMem = dma::Coherent<[u8]>; - -fn align_up(v: usize, a: usize) -> usize { - (v + a - 1) & !(a - 1) -} - -fn memcpy_to_iomem(iomem: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { - // SAFETY: - // as_slice_mut() checks that off and src.len() are whithin iomem's limits. - // memcpy_to_iomem is only called from within probe() ansuring there are no - // concurrent read and write accesses to the same region while the slice is - // alive per as_slice_mut()'s requiremnts. - unsafe { - let target = &mut iomem.as_mut()[off..off + src.len()]; - target.copy_from_slice(src); - } - Ok(()) -} - -fn build_shmem(dev: &platform::Device) -> Result { - let fwnode = dev.as_ref().fwnode().ok_or(EIO)?; - let mut iomem = dma::Coherent::::zeroed_slice(dev.as_ref(), SHMEM_SIZE, GFP_KERNEL)?; - - let panic_offset = 0x4000; - let panic_size = 0x8000; - memcpy_to_iomem(&mut iomem, panic_offset, &1u32.to_le_bytes())?; - - let lpol_offset = panic_offset + panic_size; - let lpol_prop_name = c"local-policy-manifest"; - let lpol_prop_size = fwnode.property_count_elem::(lpol_prop_name)?; - let lpol = fwnode - .property_read_array_vec(lpol_prop_name, lpol_prop_size)? - .required_by(dev.as_ref())?; - memcpy_to_iomem( - &mut iomem, - lpol_offset, - &(lpol_prop_size as u32).to_le_bytes(), - )?; - memcpy_to_iomem(&mut iomem, lpol_offset + 4, &lpol)?; - let lpol_size = align_up(lpol_prop_size + 4, 0x4000); - - let ibot_offset = lpol_offset + lpol_size; - let ibot_prop_name = c"iboot-manifest"; - let ibot_prop_size = fwnode.property_count_elem::(ibot_prop_name)?; - let ibot = fwnode - .property_read_array_vec(ibot_prop_name, ibot_prop_size)? - .required_by(dev.as_ref())?; - memcpy_to_iomem( - &mut iomem, - ibot_offset, - &(ibot_prop_size as u32).to_le_bytes(), - )?; - memcpy_to_iomem(&mut iomem, ibot_offset + 4, &ibot)?; - let ibot_size = align_up(ibot_prop_size + 4, 0x4000); - - memcpy_to_iomem(&mut iomem, 0, b"CNIP")?; - memcpy_to_iomem(&mut iomem, 4, &(panic_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 8, &(panic_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 16, b"OPLA")?; - memcpy_to_iomem(&mut iomem, 16 + 4, &(lpol_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 16 + 8, &(lpol_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 32, b"IPIS")?; - memcpy_to_iomem(&mut iomem, 32 + 4, &(ibot_size as u32).to_le_bytes())?; - memcpy_to_iomem(&mut iomem, 32 + 8, &(ibot_offset as u32).to_le_bytes())?; - - memcpy_to_iomem(&mut iomem, 48, b"llun")?; - Ok(iomem) +const SETTLE_MS: time::Msecs = 200; +const FIRST_RESPONSE_MS: time::Msecs = 3000; + +const SETTLE_WORK_ID: u64 = 1; + +const ENROL_WORK_ID: u64 = 2; +static_assert!(ENROL_WORK_ID != SETTLE_WORK_ID && ENROL_WORK_ID != 0); +const VERIFY_WORK_ID: u64 = 3; +static_assert!(VERIFY_WORK_ID != ENROL_WORK_ID && VERIFY_WORK_ID != SETTLE_WORK_ID); +static_assert!(VERIFY_WORK_ID != 0); + +const EXCHANGE_TIMEOUT_MS: time::Msecs = 15000; + +const PHASE_ATTACH: u32 = 0; +const PHASE_EXCHANGE: u32 = 1; +const PHASE_READY: u32 = 2; + +const OOL_SIZE_XARM: usize = 0x8000; + +const OOL_SIZE_SBIO: usize = 0x4000; + +const OOL_SIZE_SCRD: usize = 0x4000; + +const SBIO_TIMEOUT_MS: time::Msecs = 5000; + +const SKS_ALLOC: usize = 0x8000; + +const SKS_SECRET_LEN: usize = 32; +const SKS_MAX_CAPTURE: usize = 8; + +const SCRD_MAX_CAPTURE: usize = 8; + +const SCRD_TIMEOUT_MS: time::Msecs = 2000; + +const SKS_TIMEOUT_MS: time::Msecs = 2000; + +const SKS_TIMEOUT_PER_KIB_MS: time::Msecs = 6000; + +const SKS_TIMEOUT_MAX_MS: time::Msecs = 30_000; + +const SKS_MAX_ABANDONED: usize = 8; + +const fn sks_timeout_for(image_len: usize) -> time::Msecs { + let kib = image_len / 1024; + let kib = if kib > u16::MAX as usize { + u16::MAX as time::Msecs + } else { + kib as time::Msecs + }; + let scaled = SKS_TIMEOUT_MS.saturating_add(SKS_TIMEOUT_PER_KIB_MS.saturating_mul(kib)); + if scaled > SKS_TIMEOUT_MAX_MS { + SKS_TIMEOUT_MAX_MS + } else { + scaled + } } -#[pin_data] -struct SepReceiveWork { - data: Arc, - msg: Message, - #[pin] - work: Work, +static_assert!(sks_timeout_for(64) == SKS_TIMEOUT_MS); +static_assert!(sks_timeout_for(1508) > SKS_TIMEOUT_MS); +static_assert!(sks_timeout_for(usize::MAX) == SKS_TIMEOUT_MAX_MS); + +const SKS_MAX_SET_ASIDE: u32 = 16; + +const DMA_RING_SIZE: usize = 4 * (1 << 12); + +const OOL_WRITE_POLL_MS: u32 = 1; +const OOL_WRITE_POLL_ATTEMPTS: u32 = 200; + +const OOL_POISON_INBOUND: u8 = 0xA5; +const OOL_POISON_OUTBOUND: u8 = 0x5A; + +const PROTECTED_DATA_AVAILABLE: bool = true; + +const RNG_MAX_WORDS_PER_READ: usize = 16; + +extern "C" { + fn sep_cancel_work_sync(work: *mut c_void); + fn sep_cancel_delayed_work_sync(work: *mut c_void); } -impl_has_work! { - impl HasWork for SepReceiveWork { self.work } +const MAX_ENDPOINTS: usize = 64; + +#[derive(Clone, Copy)] +struct Endpoint { + id: u8, } -impl SepReceiveWork { - fn new(data: Arc, msg: Message) -> Result> { - Arc::pin_init( - pin_init!(SepReceiveWork { - data, - msg, - work <- new_work!("SepReceiveWork::work"), - }), - GFP_ATOMIC, - ) +impl Endpoint { + fn new(id: u8) -> Self { + Endpoint { + id, + } } } -impl WorkItem for SepReceiveWork { - type Pointer = Arc; +struct OolPair { + endpoint: u8, + allocated: usize, + declared_in: usize, + declared_out: usize, + inbound: shmem::ShMem, + outbound: shmem::ShMem, + registered: bool, +} + +fn sks_declared_sizes() -> (usize, usize) { + (0x8000, 0x4000) +} + +struct Hex<'a>(&'a [u8]); - fn run(this: Arc) { - this.data.process_message(this.msg); +impl kernel::fmt::Display for Hex<'_> { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + use core::fmt::Write; + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + for (i, byte) in self.0.iter().enumerate() { + if i > 0 { + f.write_char(' ')?; + } + f.write_char(DIGITS[(*byte >> 4) as usize] as char)?; + f.write_char(DIGITS[(*byte & 0x0f) as usize] as char)?; + } + Ok(()) } } -struct FwRegionParams { - addr: u64, - size: usize, +const SKS_LOCK_STATE_VARIANT: u32 = 1; + +#[derive(Clone, Copy)] +enum LockState { + Unlocked, } -#[pin_data] -struct SepData { - dev: ARef, - #[pin] - mbox: Mutex>>, - shmem: ShMem, - region_params: FwRegionParams, - fw_mapped: Atomic, +impl LockState { + const fn wire(self) -> i32 { + match self { + LockState::Unlocked => 0, + } + } } -impl SepData { - fn new( - dev: &platform::Device, - region_params: FwRegionParams, - ) -> Result> { - Arc::pin_init( - try_pin_init!(SepData { - shmem: build_shmem(dev)?, - dev: ARef::::from(dev.as_ref()), - mbox <- new_mutex!(None), - region_params, - fw_mapped: Atomic::new(false), - }), - GFP_KERNEL, - ) +static_assert!(LockState::Unlocked.wire() == 0); + +const SKS_LOCK_STATE_FLAGS: u64 = 0; + +enum SbioOutcome { + Ok(KVec), + // status 0x01 + PrerequisiteMissing, + // status 0x16; NOT "malformed" — that mapping is another service's status space + Status16, + Other, +} + +const BRINGUP_FRESH: u32 = 0; +const BRINGUP_IDENTIFIED: u32 = 1; +const BRINGUP_ESTABLISHED: u32 = 2; + +pub(crate) struct PatchLoaded(()); + +pub(crate) struct ParametersApplied(()); + +pub(crate) struct CalibrationBlob(kernel::firmware::Firmware); + +impl CalibrationBlob { + fn new(fw: kernel::firmware::Firmware) -> CalibrationBlob { + CalibrationBlob(fw) } - fn start(&self) -> Result<()> { - self.mbox.lock().as_ref().unwrap().send( - Message { - msg0: EP_BOOT | (MSG_BOOT_TZ0 << MSG_TYPE_SHIFT), - msg1: 0, - }, - false, - ) + + pub(crate) fn bytes(&self) -> &[u8] { + self.0.data() } - fn load_fw_and_shmem(&self) -> Result<()> { - let fw_addr = unsafe { - let res = bindings::dma_map_resource( - self.dev.as_raw(), - self.region_params.addr, - self.region_params.size, - bindings::dma_data_direction_DMA_TO_DEVICE, - 0, - ); - if bindings::dma_mapping_error(self.dev.as_raw(), res) != 0 { - dev_err!(self.dev, "Failed to map firmware"); - return Err(ENOMEM); - } - self.fw_mapped.store(true, Relaxed); - res >> IOVA_SHIFT - }; - let guard = self.mbox.lock(); - let mbox = guard.as_ref().unwrap(); - mbox.send( - Message { - msg0: EP_BOOT | (MSG_BOOT_IMG4 << MSG_TYPE_SHIFT) | (fw_addr << MSG_DATA_SHIFT), - msg1: 0, - }, - false, - )?; - let shm_addr = self.shmem.dma_handle() >> IOVA_SHIFT; - mbox.send( - Message { - msg0: EP_SHMEM | (MSG_SET_SHMEM << MSG_TYPE_SHIFT) | (shm_addr << MSG_DATA_SHIFT), - msg1: 0, - }, - false, - )?; - Ok(()) +} + +struct EnrolMaterial { + special: crate::sks::SpecialHandle, + secret: Secret, +} + +struct ImageContext<'a> { + sep: &'a SepData, +} + +impl Drop for ImageContext<'_> { + fn drop(&mut self) { + let op = crate::sbio::sbio_image_cleanup(); + let _ = self.sep.sbio_call(&op); } - fn process_boot_msg(&self, msg: Message) { - let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; - match ty { - MSG_BOOT_TZ0_ACK1 => {} - MSG_BOOT_TZ0_ACK2 => { - let res = self.load_fw_and_shmem(); - if let Err(e) = res { - dev_err!(self.dev, "Unable to load firmware: {:?}", e); - } - } - MSG_BOOT_IMG4_ACK => {} - _ => { - dev_err!(self.dev, "Unknown boot message type: {}", ty); - } - } +} + +struct OpenEnrolment<'a> { + sep: &'a SepData, + armed: bool, +} + +impl OpenEnrolment<'_> { + fn completed(mut self) { + self.armed = false; + self.sep.enrol_open.store(false, Relaxed); } - fn process_discover_msg(&self, msg: Message) { - let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; - //let data = (msg.msg0 >> MSG_DATA_SHIFT) as u32; - //let param = (msg.msg0 >> MSG_PARAM_SHIFT) & MSG_PARAM_MASK; - match ty { - MSG_ADVERTISE_EP => { - /*dev_info!( - self.dev, - "Got endpoint {:?} at {}", - core::str::from_utf8(&data.to_be_bytes()), - param - );*/ - } - _ => { - //dev_warn!(self.dev, "Unknown discovery message type: {}", ty); - } +} + +impl Drop for OpenEnrolment<'_> { + fn drop(&mut self) { + if !self.armed { + return; } + let _ = self.sep.sbio_call(&crate::sbio::sbio_cancel_operation()); + self.sep.enrol_open.store(false, Relaxed); } - fn process_message(&self, msg: Message) { - let ep = msg.msg0 & MSG_EP_MASK; - match ep { - EP_BOOT => self.process_boot_msg(msg), - EP_DISCOVER => self.process_discover_msg(msg), - _ => {} // dev_warn!(self.dev, "Message from unknown endpoint: {}", ep), +} + +enum ImageOutcome { + // `complete` flag from 0xbfe + Progress { + stage: u32, + percent: u32, + complete: bool, + has_template: bool, + }, + Retry, + NoFinger, + Failed(u32), +} + +enum CaptureWait { + Ready(u32), + Timeout, + Fault(u8), + Abandon, +} + +const ENROL_STATUS_SENSOR: u32 = 1; +const ENROL_STATUS_ENCLAVE: u32 = 2; +const ENROL_STATUS_TOO_MANY: u32 = 3; +const ENROL_STATUS_TIMEOUT: u32 = 4; +#[derive(Clone, Copy)] +enum LoadAnswer { + StatusZero, + ColdTransition, + // status 0x101: SEP re-activated the component from its own xART records + AlreadyActive, +} + +impl LoadAnswer { + fn describe(self) -> &'static CStr { + match self { + LoadAnswer::StatusZero => c"status 0", + LoadAnswer::ColdTransition => c"the cold-transition status 0x8002", + LoadAnswer::AlreadyActive => c"the already-active status 0x101", } } - fn remove(&self) { - *self.mbox.lock() = None; - if self.fw_mapped.load(Relaxed) { - unsafe { - bindings::dma_unmap_resource( - self.dev.as_raw(), - self.region_params.addr, - self.region_params.size, - bindings::dma_data_direction_DMA_TO_DEVICE, - 0, - ); - } - } +} + +enum RestoreOutcome { + Restored, + AlreadyActive, + NoStoredFile, + Failed, + // 0x8002 cold-transition; tolerated only for the lockout record, never a catacomb + EmptyTolerated, +} + +const PRIVATE_TYPE_CATACOMB_MASTER: u8 = 0xF2; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xF3; +const PRIVATE_TYPE_CATACOMB_USER: u8 = 0xF4; +const PRIVATE_TYPE_LOCKOUT: u8 = 0xF5; +// 0xF1 is the identity index's type; a colliding type silently mis-restores +static_assert!(PRIVATE_TYPE_CATACOMB_MASTER != 0xF1); +static_assert!(PRIVATE_TYPE_CATACOMB_OWNER != PRIVATE_TYPE_CATACOMB_MASTER); +static_assert!(PRIVATE_TYPE_CATACOMB_USER != PRIVATE_TYPE_CATACOMB_OWNER); +static_assert!(PRIVATE_TYPE_CATACOMB_USER != PRIVATE_TYPE_CATACOMB_MASTER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_USER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_OWNER); +static_assert!(PRIVATE_TYPE_LOCKOUT != PRIVATE_TYPE_CATACOMB_MASTER); + +const ENROL_STATUS_UNFILED: u32 = 6; +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_RETRY); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_ENCLAVE); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_SENSOR); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_TIMEOUT); +static_assert!(ENROL_STATUS_UNFILED != ENROL_STATUS_TOO_MANY); + +const ENROL_STATUS_RETRY: u32 = 5; +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_TIMEOUT); +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_ENCLAVE); +static_assert!(ENROL_STATUS_RETRY != ENROL_STATUS_SENSOR); + +const ENROL_MAX_CAPTURES: u32 = 12; + +const ENROL_POLL_MS: u32 = 2; + +// Interrupt-driven capture blocks on the data-ready line rather than polling +// SPI status. This is the maximum a single wait blocks before re-reading status +// as a backstop; the interrupt wakes it far sooner when a frame is ready. +const ENROL_IRQ_WAIT_MS: u32 = 250; + +const ENROL_CAPTURE_TIMEOUT_MS: u32 = 60_000; +const ENROL_POLL_ATTEMPTS: u32 = ENROL_CAPTURE_TIMEOUT_MS / ENROL_POLL_MS; +static_assert!(ENROL_POLL_ATTEMPTS * ENROL_POLL_MS == ENROL_CAPTURE_TIMEOUT_MS); + +const ENROL_REPOSITION_MS: u32 = 1800; + +const MATCH_SETTLE_MS: u32 = ENROL_REPOSITION_MS; + +const ENROL_IDLE_TIMEOUT_MS: u32 = 2000; + +const PATCH_POLL_MS: u32 = 20; +const PATCH_POLL_ATTEMPTS: u32 = 250; + +const CALIBRATION_FIRMWARE: &CStr = c"apple/mesa_calibration.bin"; + +const SBIO_PROBE_USER_ID: i32 = 1000; +static_assert!(SBIO_PROBE_USER_ID >= crate::sks::SKS_DESIGNATE_USER_MIN); +static_assert!(SBIO_PROBE_USER_ID == bio::ENROL_USER_ID); +static_assert!(SBIO_PROBE_USER_ID > 0); + +const SKS_LOAD_REPLY_LEN: usize = 8; + +#[derive(Clone, Copy)] + +#[must_use] +struct Healthy(()); + +struct SksRequest { + name: &'static CStr, + msg: Message, + img: image::RequestImage, +} + +struct SksOutcome { + reply: crate::sks::SksReply, + response: Secret, +} + +pub(crate) struct Secret(pub(crate) KVec); + +impl Secret { + pub(crate) fn empty() -> Secret { + Secret(KVec::new()) } } -impl MailCallback for SepData { - type Data = Arc; - fn recv_message(data: ::Borrowed<'_>, msg: Message) { - let work = SepReceiveWork::new(data.into(), msg); - if let Ok(work) = work { - let res = workqueue::system().enqueue(work); - if res.is_err() { - dev_err!( - data.dev, - "Unable to schedule work item for message {}", - msg.msg0 - ); - } - } else { - dev_err!( - data.dev, - "Unable to allocate work item for message {}", - msg.msg0 - ); - } +impl core::ops::Deref for Secret { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.0 } } -unsafe impl Send for SepData {} -unsafe impl Sync for SepData {} +impl Drop for Secret { + fn drop(&mut self) { + image::wipe(&mut self.0); + } +} -struct SepDriver(Arc); +#[derive(Clone, Copy)] +struct Abandoned { + selector: u8, + seq: u8, + label: &'static CStr, + waited_ms: u64, +} -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - (), - [(of::DeviceId::new(c"apple,sep"), ())] -); +struct SksProbe { + active: bool, + captured: KVec, + abandoned: [Option; SKS_MAX_ABANDONED], + abandoned_next: usize, +} -impl platform::Driver for SepDriver { - type IdInfo = (); +impl SksProbe { + fn new() -> Self { + SksProbe { + active: false, + captured: KVec::new(), + abandoned: [None; SKS_MAX_ABANDONED], + abandoned_next: 0, + } + } +} - const OF_ID_TABLE: Option> = Some(&OF_TABLE); +struct ScrdProbe { + active: bool, + captured: KVec, +} - fn probe( - pdev: &platform::Device, - _info: Option<&()>, - ) -> impl PinInit { - let of = pdev.as_ref().of_node().ok_or(EIO)?; - let res = of.reserved_mem_region_to_resource_byname(c"sepfw")?; - let data = SepData::new( - pdev, - FwRegionParams { - addr: res.start(), - size: res.size().try_into()?, - }, - )?; - *data.mbox.lock() = Some(Mailbox::new_byname(pdev.as_ref(), c"mbox", data.clone())?); - data.start()?; - Ok(Self(data)) +impl ScrdProbe { + fn new() -> Self { + ScrdProbe { + active: false, + captured: KVec::new(), + } } } -impl Drop for SepDriver { - fn drop(&mut self) { - self.0.remove(); +struct XarmState { + deferred_query: Option, + os_uuid: Option<[u8; 16]>, +} + +impl XarmState { + fn new() -> Self { + XarmState { + deferred_query: None, + os_uuid: None, + } } } -module_platform_driver! { - type: SepDriver, - name: "apple_sep", - description: "Secure enclave processor stub driver", - license: "Dual MIT/GPL", +struct EndpointTable { + eps: KVec, +} + +impl EndpointTable { + fn new() -> Self { + EndpointTable { + eps: KVec::new(), + } + } + + fn slot(&mut self, id: u8) -> Result { + if let Some(i) = self.eps.iter().position(|e| e.id == id) { + return Ok(i); + } + if self.eps.len() >= MAX_ENDPOINTS { + return Err(ENOSPC); + } + self.eps.push(Endpoint::new(id), GFP_KERNEL)?; + Ok(self.eps.len() - 1) + } +} + + +struct MachineRefKey { + blob: KVec, + pub_raw: KVec, +} + +/// One mapped firmware reserved region, unmapped exactly once in `remove()`. +struct FirmwareMap { + size: usize, + iova: u64, +} + +#[pin_data] +struct SepData { + dev: ARef, + + profile: &'static profile::PlatformProfile, + + #[pin] + fw_map: Mutex>, + + #[pin] + mbox: Mutex>>, + + #[pin] + shmem: Mutex>, + + #[pin] + endpoints: Mutex, + + #[pin] + control: Mutex, + #[pin] + control_wq: CondVar, + + sks_wedged: Atomic, + + sks_seq: Atomic, + + // u32, not u8: the kernel's Rust atomics have no u8 AtomicType + phase: Atomic, + + #[pin] + ool_xarm: Mutex>, + + #[pin] + ool_sbio: Mutex>, + + #[pin] + dma_ring: Mutex>, + + #[pin] + sbio_rx: Mutex, + + #[pin] + sbio_wq: CondVar, + + sbio_ready: Atomic, + + templates_restored: Atomic, + + sensor_calibrated: Atomic, + + enrol_open: Atomic, + + #[pin] + enrol_material: Mutex>, + + #[pin] + enrol_identity_candidates: Mutex>, + + last_capture_end_ns: Atomic, + + device_view_synced: Atomic, + + keybag_designated: Atomic, + + bringup_started: Atomic, + touchid_started: Atomic, + touchid_failed: Atomic, + + bringup: Atomic, + + #[pin] + ool_sks: Mutex>, + + #[pin] + sks_exchange_lock: Mutex<()>, + #[pin] + sks_probe: Mutex, + #[pin] + sks_wq: CondVar, + + #[pin] + ool_scrd: Mutex>, + #[pin] + scrd_probe: Mutex, + #[pin] + scrd_wq: CondVar, + + sensor_present: Atomic, + + #[pin] + bio_session: Mutex, + + #[pin] + bio_index: Mutex, + + #[pin] + bio_dev: Mutex>, + + #[pin] + store: Mutex>, + + #[pin] + host_store: Mutex>, + + #[pin] + xarm: Mutex, + + #[pin] + rng: Mutex>, + + #[pin] + machine_refkey: Mutex>, + + #[pin] + fv_volumes: Mutex>, + + rng_shutdown: Atomic, + + rng_failures: Atomic, + + rx: rxring::RxRing, + + rx_count: Atomic, + settle_mark: Atomic, + settle_idle_ticks: Atomic, + + registered: Atomic, + + shutting_down: Atomic, + + #[pin] + rx_work: Work, + + #[pin] + settle_work: DelayedWork, + + #[pin] + enrol_work: Work, + + #[pin] + verify_work: Work, +} + +impl_has_work! { + impl HasWork for SepData { self.rx_work } + impl HasWork for SepData { self.enrol_work } + impl HasWork for SepData { self.verify_work } +} + +impl_has_delayed_work! { + impl HasDelayedWork for SepData { self.settle_work } +} + +// SAFETY: every field is either internally synchronised (the mutexes, the +// atomics, the lock-free ring) or immutable after construction. The DMA buffer +// is only touched in probe, before the SEP has been told it exists, and at +// unbind, after the mailbox has been stopped. +unsafe impl Send for SepData {} +// SAFETY: see above. +unsafe impl Sync for SepData {} + +impl SepData { + fn new(pdev: &platform::Device) -> Result> { + let profile = profile::detect()?; + let built = shmem::build(pdev, profile.shmem_capacity, profile.shmem_first_item)?; + let dev: &device::Device = pdev.as_ref(); + dev_info!(dev, "SEP platform profile: {}\n", profile.name); + + let buf = built.buf; + + let ool_xarm = Self::alloc_ool( + dev, + xarm::EP_XARM, + OOL_SIZE_XARM, + (OOL_SIZE_XARM, OOL_SIZE_XARM), + )?; + let ool_sbio = Self::alloc_ool( + dev, + proto::EP_SBIO, + OOL_SIZE_SBIO, + (OOL_SIZE_SBIO, OOL_SIZE_SBIO), + )?; + let sks_declared = sks_declared_sizes(); + let ool_sks = Self::alloc_ool(dev, proto::EP_SKS, SKS_ALLOC, sks_declared)?; + let ool_scrd = Self::alloc_ool( + dev, + proto::EP_SCRD, + OOL_SIZE_SCRD, + (OOL_SIZE_SCRD, OOL_SIZE_SCRD), + )?; + let dma_ring = dma::Coherent::::zeroed_slice(dev, DMA_RING_SIZE, GFP_KERNEL)?; + + let xart_writes = *module_parameters::xart_writes.value() != 0; + let xart_start = *module_parameters::xart_start_sector.value(); + let store = match xart_store::Store::open_owner(xart_writes, xart_start) { + Ok(store) => { + let (slots, records, revision, malformed, duplicates, repaired, writable) = + store.summary(); + let base = store.base(); + dev_info!( + dev, + "xART: in-kernel raw-extent owner (base {:#x}); {} slots, {} live records, max revision {}, {} malformed, {} duplicate, {} repaired; writes {}\n", + base, + slots, + records, + revision, + malformed, + duplicates, + repaired, + if writable { "ENABLED" } else { "disabled" } + ); + Some(store) + } + Err(e) => { + dev_err!( + dev, + "xART gigalocker in the iBoot container is unavailable or invalid: {:?}\n", + e + ); + return Err(e); + } + }; + + let mut host_store = match store::Store::open() { + Ok(store) => Some(store), + Err(e) => { + dev_warn!(dev, "Linux host-state store unavailable: {:?}\n", e); + None + } + }; + + let bio_index = match host_store.as_mut().map(bio::IdentityIndex::load) { + Some(Ok(index)) => index, + Some(Err(_)) => bio::IdentityIndex::new(), + None => bio::IdentityIndex::new(), + }; + + Arc::pin_init( + try_pin_init!(SepData { + dev: ARef::::from(dev), + profile, + fw_map <- new_mutex!(None), + mbox <- new_mutex!(None), + shmem <- new_mutex!(Some(buf)), + endpoints <- new_mutex!(EndpointTable::new()), + control <- new_mutex!(control::ControlState::new()), + control_wq <- new_condvar!("SepData::control_wq"), + sks_seq: Atomic::new(0), + sks_wedged: Atomic::new(0), + bringup: Atomic::new(BRINGUP_FRESH), + keybag_designated: Atomic::new(false), + bringup_started: Atomic::new(false), + touchid_started: Atomic::new(false), + touchid_failed: Atomic::new(false), + enrol_open: Atomic::new(false), + sensor_calibrated: Atomic::new(false), + templates_restored: Atomic::new(false), + enrol_material <- new_mutex!(None), + enrol_identity_candidates <- new_mutex!(KVec::new()), + last_capture_end_ns: Atomic::new(0), + device_view_synced: Atomic::new(false), + phase: Atomic::new(PHASE_ATTACH), + ool_xarm <- new_mutex!(Some(ool_xarm)), + ool_sbio <- new_mutex!(Some(ool_sbio)), + ool_sks <- new_mutex!(Some(ool_sks)), + sks_exchange_lock <- new_mutex!(()), + sks_probe <- new_mutex!(SksProbe::new()), + sks_wq <- new_condvar!("SepData::sks_wq"), + ool_scrd <- new_mutex!(Some(ool_scrd)), + scrd_probe <- new_mutex!(ScrdProbe::new()), + scrd_wq <- new_condvar!("SepData::scrd_wq"), + dma_ring <- new_mutex!(Some(dma_ring)), + sbio_rx <- new_mutex!(transfer::Reassembly::new()), + sbio_wq <- new_condvar!("SepData::sbio_wq"), + sbio_ready: Atomic::new(false), + sensor_present: Atomic::new(false), + bio_session <- new_mutex!(bio::Session::new()), + bio_index <- new_mutex!(bio_index), + bio_dev <- new_mutex!(None), + store <- new_mutex!(store), + host_store <- new_mutex!(host_store), + xarm <- new_mutex!(XarmState::new()), + rng <- new_mutex!(None), + machine_refkey <- new_mutex!(None), + fv_volumes <- new_mutex!(KVec::new()), + rng_shutdown: Atomic::new(false), + rng_failures: Atomic::new(0), + rx: rxring::RxRing::new(), + rx_count: Atomic::new(0), + settle_mark: Atomic::new(0), + settle_idle_ticks: Atomic::new(0), + registered: Atomic::new(false), + shutting_down: Atomic::new(false), + rx_work <- new_work!("SepData::rx_work"), + enrol_work <- new_work!("SepData::enrol_work"), + verify_work <- new_work!("SepData::verify_work"), + settle_work <- new_delayed_work!("SepData::settle_work"), + }), + GFP_KERNEL, + ) + } + + /// Bring the shared-memory table to the SEP using the profile's bootstrap. + fn attach(&self, sep_node: &dt::DtNode) -> Result<()> { + match self.profile.bootstrap { + profile::Bootstrap::WarmRegister => self.attach_warm(sep_node), + profile::Bootstrap::Boot => self.attach_m1(), + } + } + + /// Warm attach: the firmware left a running SEP, so one registration message + /// hands it the shared-memory table. + fn attach_warm(&self, sep_node: &dt::DtNode) -> Result<()> { + let (iova, size) = { + let guard = self.shmem.lock(); + let buf = guard.as_ref().ok_or(EINVAL)?; + (buf.dma_handle(), buf.len()) + }; + + let msg = proto::shmem_registration(iova, size)?; + + + if dt::registration_already_sent(sep_node) { + return Err(EBUSY); + } + + if let Err(e) = dt::mark_registration_sent(sep_node, iova) { + dev_err!( + self.dev, + "could not record the registration marker ({:?}); refusing to send, since without it an rmmod/insmod cycle could send it twice and park the SEP\n", + e + ); + return Err(e); + } + self.registered.store(true, Relaxed); + + self.mbox.lock().as_ref().ok_or(EINVAL)?.send(msg, false)?; + + // no ack on 0xFE; success is the discovery burst on 0xFD + Ok(()) + } + + /// Cold boot: send TZ0 and let the boot-endpoint acknowledgements drive the + /// firmware and shared-memory handoff. + fn attach_m1(&self) -> Result<()> { + let msg = Message { + msg0: u64::from(proto::EP_BOOT) | (proto::MSG_BOOT_TZ0 << proto::MSG_TYPE_SHIFT), + msg1: 0, + }; + self.send(msg) + } + + /// Handle a message from the boot endpoint during the cold-boot handshake. + fn on_boot(&self, msg: Message) { + let ty = (msg.msg0 >> proto::MSG_TYPE_SHIFT) & 0xff; + match ty { + proto::MSG_BOOT_TZ0_ACK1 => { + dev_info!(self.dev, "boot: first TZ0 acknowledgement\n"); + } + proto::MSG_BOOT_TZ0_ACK2 => { + dev_info!( + self.dev, + "boot: TZ0 accepted; handing over firmware and shared memory\n" + ); + if let Err(e) = self.load_firmware_and_shmem() { + dev_err!( + self.dev, + "boot: firmware/shared-memory handoff failed ({:?})\n", + e + ); + } + } + proto::MSG_BOOT_IMG4_ACK => { + dev_info!( + self.dev, + "boot: IMG4 acknowledged; the SEP owns the shared-memory table\n" + ); + self.registered.store(true, Relaxed); + } + _ => { + dev_warn!( + self.dev, + "boot: unknown message type {} (msg0 {:#018x})\n", + ty, + msg.msg0 + ); + } + } + } + + /// Map the firmware reserved region once and hand the SEP the firmware + /// address (IMG4) and the shared-memory table (SET_SHMEM). + fn load_firmware_and_shmem(&self) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + + let (phys, size) = dt::reserved_region(&self.dev, self.profile.firmware_region)?; + + // Hold the mapping lock across the teardown re-check, the map and the + // store: `remove()` takes the same lock, so a mapping either exists + // before removal and is unmapped by it, or is refused here. + let mut fw_map = self.fw_map.lock(); + if fw_map.is_some() { + // One-shot per boot: a second acknowledgement cannot remap it. + return Ok(()); + } + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + // SAFETY: `self.dev` is live; the reserved region is owned by the + // firmware handoff and the mapping is retained until `remove()`. + let iova = unsafe { + let mapped = bindings::dma_map_resource( + self.dev.as_raw(), + phys, + size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + if bindings::dma_mapping_error(self.dev.as_raw(), mapped) != 0 { + return Err(ENOMEM); + } + mapped + }; + *fw_map = Some(FirmwareMap { size, iova }); + drop(fw_map); + + let msg = Message { + msg0: u64::from(proto::EP_BOOT) + | (proto::MSG_BOOT_IMG4 << proto::MSG_TYPE_SHIFT) + | ((iova >> proto::IOVA_SHIFT) << proto::MSG_DATA_SHIFT), + msg1: 0, + }; + self.send(msg)?; + + let shm = { + let guard = self.shmem.lock(); + let buf = guard.as_ref().ok_or(EINVAL)?; + buf.dma_handle() + }; + let msg = Message { + msg0: u64::from(proto::EP_SHMEM) + | (proto::MSG_SET_SHMEM << proto::MSG_TYPE_SHIFT) + | ((shm >> proto::IOVA_SHIFT) << proto::MSG_DATA_SHIFT), + msg1: 0, + }; + // The peer may know the address even if this send fails ambiguously, so + // the buffer stays retained: `remove()` must never free shared memory + // the SEP was handed. + self.registered.store(true, Relaxed); + self.send(msg)?; + Ok(()) + } + + /// Release the firmware mapping if one is live. Called by `remove()` so a + /// mapping created by a racing boot message cannot outlive the driver. + fn release_firmware_mapping(&self) { + if let Some(mapped) = self.fw_map.lock().take() { + // SAFETY: created by `dma_map_resource` with these exact parameters + // and unmapped exactly once. + unsafe { + bindings::dma_unmap_resource( + self.dev.as_raw(), + mapped.iova, + mapped.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + } + } + } + + fn send(&self, msg: Message) -> Result<()> { + self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) + } + + + fn control_request(&self, op: &proto::ControlOp) -> Result> { + let (idx, tag) = self.control.lock().alloc()?; + let msg = proto::encode_control(op, tag); + + if let Err(e) = self.send(msg) { + self.control.lock().abandon(idx); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); + let mut guard = self.control.lock(); + loop { + if let Some(reply) = guard.take_reply(idx) { + guard.release(idx); + drop(guard); + if reply.tag != tag { + return Err(EIO); + } + return Ok(Some(reply)); + } + + if remaining == 0 { + let tag = guard.abandon(idx); + let retired = guard.retired_count(); + drop(guard); + if op.expects_reply() { + dev_warn!( + self.dev, + "control <- {}: no reply within {} ms; tag 0x{:02x} retired ({} retired in total)\n", + op.name(), + op.timeout_ms(), + tag, + retired + ); + } + return Ok(None); + } + + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn get_entropy_word(&self) -> Result { + let op = proto::op_get_entropy(); + if !op.uses_reserved_tag() { + return Err(EINVAL); + } + + self.control.lock().entropy_begin()?; + + let msg = proto::encode_control(&op, proto::TAG_ENTROPY); + if let Err(e) = self.send(msg) { + self.control.lock().entropy_end(); + return Err(e); + } + + let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); + let mut guard = self.control.lock(); + let result = loop { + if let Some(value) = guard.entropy_take() { + break Ok(value); + } + if self.rng_shutdown.load(Relaxed) { + break Err(ECANCELED); + } + if remaining == 0 { + break Err(ETIMEDOUT); + } + match self + .control_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + }; + guard.entropy_end(); + result + } + + fn control_survey(&self) { + + for param in [0x00u8, 0x01, 0xff] { + let _ = self.control_request(&proto::op_nop(param)); + } + + let _ = self.control_request(&proto::op_security_mode()); + + let mut words = [0u32; 4]; + let mut got = 0; + for w in words.iter_mut() { + match self.get_entropy_word() { + Ok(v) => { + *w = v; + got += 1; + } + Err(_) => { + break; + } + } + } + + if got == words.len() { + match self.register_hwrng() { + Ok(()) => {}, + Err(e) => dev_err!(self.dev, "hwrng: registration failed: {:?}\n", e), + } + } else { + dev_err!( + self.dev, + "hwrng: not registering, the source did not answer during the survey\n" + ); + } + + let _state = self.control.lock(); + } + + + fn alloc_ool( + dev: &device::Device, + endpoint: u8, + allocated: usize, + declared: (usize, usize), + ) -> Result { + if declared.0 > allocated || declared.1 > allocated { + return Err(EINVAL); + } + + let inbound = dma::Coherent::::zeroed_slice(dev, allocated, GFP_KERNEL)?; + let outbound = dma::Coherent::::zeroed_slice(dev, allocated, GFP_KERNEL)?; + + // SAFETY: nothing is registered yet, so the SEP does not know these + // buffers exist and cannot be touching them. + unsafe { + inbound.as_mut().fill(OOL_POISON_INBOUND); + outbound.as_mut().fill(OOL_POISON_OUTBOUND); + } + + Ok(OolPair { + endpoint, + allocated, + declared_in: declared.0, + declared_out: declared.1, + inbound, + outbound, + registered: false, + }) + } + + fn control_request_required(&self, op: &proto::ControlOp) -> Result { + match self.control_request(op)? { + Some(reply) => Ok(reply), + None => Err(ETIMEDOUT), + } + } + + fn register_ool(&self, slot: &Mutex>) -> Result<()> { + let guard = slot.lock(); + if guard.as_ref().is_some_and(|b| b.registered) { + return Ok(()); + } + + let (endpoint, allocated, declared_in, declared_out, in_iova, out_iova) = { + let b = guard.as_ref().ok_or(EINVAL)?; + ( + b.endpoint, + b.allocated, + b.declared_in, + b.declared_out, + b.inbound.dma_handle(), + b.outbound.dma_handle(), + ) + }; + + if declared_in > allocated || declared_out > allocated { + return Err(EINVAL); + } + + // drop the buffer lock before blocking control requests; the drain path takes it + drop(guard); + + self.control_request_required(&proto::op_ool_inbound_size(endpoint, declared_in as u32))?; + self.control_request_required(&proto::op_ool_inbound_addr(endpoint, in_iova))?; + self.control_request_required(&proto::op_ool_outbound_size(endpoint, declared_out as u32))?; + self.control_request_required(&proto::op_ool_outbound_addr(endpoint, out_iova))?; + + { + // deref first, or the guard's inherent as_mut wins over Option::as_mut + let mut guard = slot.lock(); + let pair: &mut Option = &mut guard; + if let Some(b) = pair.as_mut() { + b.registered = true; + } + } + Ok(()) + } + + fn with_store(&self, f: impl FnOnce(&mut xart_store::Store) -> R) -> Option { + let mut guard = self.store.lock(); + let store: &mut Option = &mut guard; + store.as_mut().map(f) + } + + fn with_host_store(&self, f: impl FnOnce(&mut store::Store) -> R) -> Option { + let mut guard = self.host_store.lock(); + let store: &mut Option = &mut guard; + store.as_mut().map(f) + } + + fn prepare_os_uuid(&self) { + let hi = *module_parameters::os_uuid_hi.value(); + let lo = *module_parameters::os_uuid_lo.value(); + if hi != 0 || lo != 0 { + let mut uuid = [0u8; 16]; + uuid[..8].copy_from_slice(&hi.to_be_bytes()); + uuid[8..].copy_from_slice(&lo.to_be_bytes()); + self.xarm.lock().os_uuid = Some(uuid); + dev_info!( + self.dev, + "xART: using explicit OS UUID {:016x}-{:016x}\n", + hi, + lo + ); + return; + } + + if let Some(uuid) = dt::preboot_uuid() { + self.xarm.lock().os_uuid = Some(uuid); + dev_info!(self.dev, "xART: using /chosen/apfs-preboot-uuid\n"); + } else { + self.xarm.lock().os_uuid = None; + dev_warn!(self.dev, "xART: /chosen/apfs-preboot-uuid is unavailable\n"); + } + } + + fn on_xarm(&self, msg: Message) { + let req = crate::xarm::decode_xarm(&msg); + + if xarm::is_silent(req.opcode) { + return; + } + + let ready = self.ool_xarm.lock().as_ref().is_some_and(|b| b.registered); + + if !ready { + if !xarm::needs_buffers(req.opcode) { + self.xarm.lock().deferred_query = Some(req.tag); + return; + } + + self.fail_xarm(req.tag); + return; + } + + self.service_xarm(&req); + } + + fn service_xarm(&self, req: &crate::xarm::XarmRequest) { + let want = req.length as usize; + if want > OOL_SIZE_XARM { + self.fail_xarm(req.tag); + return; + } + + if want > 0 && !self.ool_await_written(&self.ool_xarm, 0, want) { + self.fail_xarm(req.tag); + return; + } + let payload = match self.ool_read(&self.ool_xarm, 0, want) { + Ok(p) => p, + Err(_) => { + self.fail_xarm(req.tag); + return; + } + }; + + let mut staging = KVec::new(); + if staging.resize(OOL_SIZE_XARM, 0u8, GFP_KERNEL).is_err() { + self.fail_xarm(req.tag); + return; + } + + let os_uuid = self.xarm.lock().os_uuid; + let serviced = self.with_store(|store| { + xarm::service( + req, + &payload, + &mut staging, + store, + PROTECTED_DATA_AVAILABLE, + os_uuid, + ) + }); + let Some(done) = serviced else { + self.fail_xarm(req.tag); + return; + }; + + if done.reply_bytes > 0 { + if self.ool_write(&self.ool_xarm, 0, &staging[..done.reply_bytes]).is_err() { + self.fail_xarm(req.tag); + return; + } + } + + self.send_xarm_reply(&done.reply); + } + + fn ool_read(&self, slot: &Mutex>, off: usize, len: usize) -> Result> { + let mut guard = slot.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + if off + len > buffers.allocated { + return Err(EINVAL); + } + + let mut payload = KVec::new(); + if len > 0 { + // SAFETY: the SEP fills this buffer while preparing a request and + // then waits for the host's reply, so it is quiescent here, and the + // driver is the only other accessor. + let src = unsafe { &buffers.outbound.as_ref()[off..off + len] }; + payload.extend_from_slice(src, GFP_KERNEL)?; + } + + Ok(payload) + } + + fn ool_await_written(&self, slot: &Mutex>, off: usize, len: usize) -> bool { + for _ in 0..OOL_WRITE_POLL_ATTEMPTS { + { + let guard = slot.lock(); + let Some(buffers) = guard.as_ref() else { + return false; + }; + if off + len > buffers.allocated { + return true; + } + // SAFETY: the region is within the allocation as checked above, + // and the driver is the only host-side accessor. + let region = unsafe { &buffers.outbound.as_ref()[off..off + len] }; + if region.iter().any(|&b| b != OOL_POISON_OUTBOUND) { + return true; + } + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(i64::from(OOL_WRITE_POLL_MS))); + } + false + } + + fn ool_write(&self, slot: &Mutex>, off: usize, bytes: &[u8]) -> Result<()> { + let guard = slot.lock(); + let buffers = guard.as_ref().ok_or(ENODEV)?; + if off + bytes.len() > buffers.allocated { + return Err(ENOSPC); + } + // SAFETY: the SEP reads this buffer only after the reply message is + // sent, which happens after this returns; the driver is the only writer. + unsafe { buffers.inbound.as_mut()[off..off + bytes.len()].copy_from_slice(bytes) }; + Ok(()) + } + + fn fail_xarm(&self, tag: u8) { + self.send_xarm_reply(&crate::xarm::XarmReply { + tag, + status: xarm::STATUS_FAILED, + length: 0, + args: [0; 3], + }); + } + + fn send_xarm_reply(&self, reply: &crate::xarm::XarmReply) { + let msg = crate::xarm::encode_xarm_reply(reply); + let _ = self.send(msg); + } + + fn release_deferred_query(&self) { + let Some(tag) = self.xarm.lock().deferred_query.take() else { + return; + }; + let mut reply = crate::xarm::XarmReply { + tag, + status: xarm::STATUS_OK, + length: 0, + args: [0; 3], + }; + reply.args[0] = u8::from(PROTECTED_DATA_AVAILABLE); + self.send_xarm_reply(&reply); + } + + + fn on_scrd(&self, msg: Message) { + let mut probe = self.scrd_probe.lock(); + if probe.active { + if probe.captured.len() < SCRD_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.scrd_wq.notify_all(); + } + } + + fn scrd_arm(&self) { + let mut probe = self.scrd_probe.lock(); + probe.captured.clear(); + probe.active = true; + } + + fn scrd_disarm(&self) -> KVec { + let mut probe = self.scrd_probe.lock(); + probe.active = false; + core::mem::take(&mut probe.captured) + } + + fn scrd_wait(&self, ms: time::Msecs, until: usize) { + let mut remaining = time::msecs_to_jiffies(ms); + let mut guard = self.scrd_probe.lock(); + loop { + if guard.captured.len() >= until || remaining == 0 { + return; + } + match self + .scrd_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + + fn scrd_zero_buffers(&self) -> Result<()> { + let mut guard = self.ool_scrd.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + // SAFETY: nothing is outstanding on this endpoint here; the enclave + // touches these only between a request going out and its reply, and the + // driver is the only other accessor. + unsafe { + buffers.inbound.as_mut()[..buffers.allocated].fill(0); + buffers.outbound.as_mut()[..buffers.allocated].fill(0); + } + Ok(()) + } + + fn scrd_transact( + &self, + _name: &'static CStr, + request: u8, + payload: &[u8], + timeout_ms: time::Msecs, + ) -> Option<(crate::scrd::ScrdReply, KVec)> { + if !self.ool_registered(&self.ool_scrd) { + return None; + } + if self.scrd_zero_buffers().is_err() { + return None; + } + if !payload.is_empty() { + if self.ool_write(&self.ool_scrd, 0, payload).is_err() { + return None; + } + } + + let msg = crate::scrd::encode_scrd(request, payload.len()); + self.scrd_arm(); + if self.send(msg).is_err() { + let _ = self.scrd_disarm(); + return None; + } + + let mut examined = 0usize; + let correlating = loop { + self.scrd_wait(timeout_ms, examined + 1); + let next = { + let probe = self.scrd_probe.lock(); + let m = probe.captured.get(examined).copied(); + if m.is_some() { + examined += 1; + } + m + }; + let Some(candidate) = next else { break None }; + let decoded = crate::scrd::decode_scrd_reply(&candidate); + if decoded.request == request { + break Some(candidate); + } + if examined >= SCRD_MAX_CAPTURE { + break None; + } + }; + + let leftover = self.scrd_disarm(); + let correlating = match correlating { + Some(m) => Some(m), + None => leftover + .iter() + .skip(examined) + .copied() + .find(|m| crate::scrd::decode_scrd_reply(m).request == request), + }; + + let raw = correlating?; + + let reply = crate::scrd::decode_scrd_reply(&raw); + let mut response = KVec::new(); + if reply.response_size > 0 { + if let Ok(bytes) = self.ool_read(&self.ool_scrd, 0, reply.response_size as usize) { + response = bytes; + } + } + Some((reply, response)) + } + + fn scrd_ready(&self) -> bool { + if self.ool_registered(&self.ool_scrd) { + return true; + } + if !self.endpoint_present(proto::EP_SCRD) { + return false; + } + if self.register_ool(&self.ool_scrd).is_err() { + return false; + } + true + } + + fn scrd_command(&self, cmd: &crate::scrd::ScrdCommand) -> Option<(i32, KVec)> { + let (reply, out) = + self.scrd_transact(cmd.name(), cmd.request(), cmd.payload(), SCRD_TIMEOUT_MS)?; + Some((reply.status, out)) + } + + fn establish_passcode_validated_context( + &self, + user: crate::sbio::UserId, + ) -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + if !self.scrd_ready() { + return None; + } + + let (special, secret) = { + let guard = self.enrol_material.lock(); + let material = guard.as_ref()?; + let mut copy = KVec::new(); + copy.extend_from_slice(&material.secret, GFP_KERNEL).ok()?; + (material.special, Secret(copy)) + }; + + let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + if status != 0 { + return None; + } + + let (status, out) = self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + return None; + } + let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; + acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); + + let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + if status != 0 { + return None; + } + + let out = self.sks_send(self.sks_req_verify_secret(special, &secret, &acm_handle))?; + if out.reply.status != 0 { + return None; + } + + let (status, out) = self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + let satisfied = out.len() >= 4 + && u32::from_le_bytes([out[0], out[1], out[2], out[3]]) != 0; + if status != 0 || !satisfied { + return None; + } + + Some(acm_handle) + } + + fn establish_scrd_match_context(&self, user: crate::sbio::UserId) { + if !self.scrd_ready() { + return; + } + let Some(du) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return; + }; + let special = du.special_handle(); + let stored = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(s)) => s, + _ => { + return; + } + }; + + let _ = (|| -> Option<[u8; crate::scrd::SCRD_ACM_HANDLE_LEN]> { + let (status, _) = self.scrd_command(&crate::scrd::scrd_initialize())?; + if status != 0 { + return None; + } + let (status, out) = + self.scrd_command(&crate::scrd::scrd_context_create_tracked(user.value()))?; + if status != 0 || out.len() < crate::scrd::SCRD_ACM_HANDLE_LEN { + return None; + } + let mut acm_handle = [0u8; crate::scrd::SCRD_ACM_HANDLE_LEN]; + acm_handle.copy_from_slice(&out[..crate::scrd::SCRD_ACM_HANDLE_LEN]); + let (status, _) = self.scrd_command(&crate::scrd::scrd_context_externalize(&acm_handle))?; + if status != 0 { + return None; + } + let out = self.sks_send(self.sks_req_verify_secret(special, stored.secret(), &acm_handle))?; + if out.reply.status != 0 { + return None; + } + self.scrd_command(&crate::scrd::scrd_verify_touchid_enrollment(&acm_handle))?; + Some(acm_handle) + })(); + } + + + fn hwrng_fill(&self, buf: *mut u8, max: usize, wait: bool) -> c_int { + if !wait { + return 0; + } + + let words = core::cmp::min(max / 4, RNG_MAX_WORDS_PER_READ); + let mut written: usize = 0; + let mut failure: Option = None; + + for i in 0..words { + if self.rng_shutdown.load(Relaxed) { + failure = Some(ENODEV); + break; + } + match self.get_entropy_word() { + Ok(value) => { + // SAFETY: the core guarantees `buf` is valid for `max` + // bytes and aligned for any type, and `i < max / 4`. + unsafe { buf.add(i * 4).cast::().write_unaligned(value) }; + written += 4; + } + Err(e) => { + failure = Some(e); + break; + } + } + } + + if written > 0 { + return written as c_int; + } + + // must return a negative errno, not 0: a blocking reader would spin on 0 + let e = failure.unwrap_or(EIO); + let n = self.rng_failures.load(Relaxed).wrapping_add(1); + self.rng_failures.store(n, Relaxed); + if n == 1 { + dev_err!( + self.dev, + "hwrng: read produced nothing ({:?}); further failures will not be logged\n", + e + ); + } + e.to_errno() + } + + fn register_hwrng(&self) -> Result<()> { + let mut guard = self.rng.lock(); + if guard.is_some() { + return Ok(()); + } + + let mut handle = hwrng::HwRngHandle::new()?; + let ctx = core::ptr::from_ref(self).cast_mut().cast::(); + + // SAFETY: `ctx` is this `SepData`, which is kept alive by the `Arc` held + // in the driver's private data. `remove()` sets `rng_shutdown`, wakes + // any blocked draw, and unregisters, and `hwrng_unregister()` blocks + // until the core has finished with the device, all before that `Arc` + // can be dropped. So the pointer cannot outlive the registration. + unsafe { handle.register(ctx, hwrng_read_trampoline) }?; + + *guard = Some(handle); + Ok(()) + } + + + fn drain(&self) -> u32 { + let mut n: u32 = 0; + while let Some(msg) = self.rx.pop() { + self.dispatch(msg); + n += 1; + } + if n > 0 { + // single writer, single instance: a load/store pair suffices + let total = self.rx_count.load(Relaxed).wrapping_add(u64::from(n)); + self.rx_count.store(total, Relaxed); + } + n + } + + + fn arm_settle(this: &Arc) { + if this.shutting_down.load(Relaxed) { + return; + } + let delay = time::msecs_to_jiffies(SETTLE_MS); + let _ = workqueue::system() + .enqueue_delayed::, SETTLE_WORK_ID>(this.clone(), delay); + } + + fn settle_tick(this: &Arc) { + match this.phase.load(Relaxed) { + PHASE_ATTACH => Self::tick_attach(this), + PHASE_EXCHANGE => Self::tick_exchange(this), + _ => {} + } + } + + fn tick_attach(this: &Arc) { + let now = this.rx_count.load(Relaxed); + + // HW: the SEP takes ~265 ms to answer a registration + if now == 0 { + let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); + this.settle_idle_ticks.store(ticks, Relaxed); + if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(FIRST_RESPONSE_MS) { + Self::arm_settle(this); + return; + } + } + + if now != this.settle_mark.load(Relaxed) { + this.settle_mark.store(now, Relaxed); + Self::arm_settle(this); + return; + } + + if this.endpoint_count() == 0 { + this.phase.store(PHASE_READY, Relaxed); + return; + } + + this.prepare_os_uuid(); + + this.control_survey(); + + if let Err(e) = this.register_ool(&this.ool_xarm) { + dev_err!( + this.dev, + "xarm: out-of-line buffer registration failed ({:?}); the persistent-state exchange cannot run\n", + e + ); + this.phase.store(PHASE_READY, Relaxed); + return; + } + + this.phase.store(PHASE_EXCHANGE, Relaxed); + this.settle_mark.store(this.rx_count.load(Relaxed), Relaxed); + this.settle_idle_ticks.store(0, Relaxed); + this.release_deferred_query(); + Self::arm_settle(this); + } + + fn tick_exchange(this: &Arc) { + let now = this.rx_count.load(Relaxed); + + if now != this.settle_mark.load(Relaxed) { + this.settle_mark.store(now, Relaxed); + this.settle_idle_ticks.store(0, Relaxed); + Self::arm_settle(this); + return; + } + + // The endpoint ladder climbs 7 -> 11 (sbio, scrd) -> 12 (the key + // store, EP 0x12). The SEP can advertise the key store a beat after it + // falls briefly quiet following scrd, so treating any count past the + // pre-exchange baseline as "done" races that last step and strands the + // boot at 11 endpoints with no key store. Wait through quiescence until + // the key store itself is advertised, bounded by the same + // EXCHANGE_TIMEOUT_MS. A boot that already has it exits immediately, so + // healthy boots are unaffected; this also drops the hardcoded endpoint + // count, which is not portable across SoCs. + if !this.endpoint_present(proto::EP_SKS) { + let ticks = this.settle_idle_ticks.load(Relaxed).wrapping_add(1); + this.settle_idle_ticks.store(ticks, Relaxed); + if ticks.saturating_mul(u64::from(SETTLE_MS)) < u64::from(EXCHANGE_TIMEOUT_MS) { + Self::arm_settle(this); + return; + } + } + + this.phase.store(PHASE_READY, Relaxed); + this.run_bringup(); + } + + fn endpoint_present(&self, id: u8) -> bool { + self.endpoints.lock().eps.iter().any(|e| e.id == id) + } + + fn endpoint_count(&self) -> usize { + self.endpoints.lock().eps.len() + } + + fn dispatch(&self, msg: Message) { + let f = proto::decode(&msg); + + match f.ep { + proto::EP_DISCOVER => self.on_discovery(msg, f), + + proto::EP_SHMEM => {}, + + proto::EP_CONTROL => self.on_control(msg, f), + + xarm::EP_XARM => self.on_xarm(msg), + + proto::EP_SBIO => self.on_sbio(msg), + + proto::EP_SKS => self.on_sks(msg), + + proto::EP_SCRD => self.on_scrd(msg), + + proto::EP_BOOT => self.on_boot(msg), + + _ep => {}, + } + } + + fn on_control(&self, msg: Message, f: proto::Fields) { + if f.ty != proto::CONTROL_REPLY_TYPE { + return; + } + + let reply = proto::ControlReply::from_message(&msg); + let disposition = self.control.lock().deliver(reply); + + match disposition { + control::Delivery::Entropy => { + self.control_wq.notify_all(); + } + control::Delivery::Matched => self.control_wq.notify_all(), + control::Delivery::Unmatched => {} + } + } + + fn on_discovery(&self, _msg: Message, f: proto::Fields) { + let mut table = self.endpoints.lock(); + match f.ty { + proto::DISCOVER_TYPE_DESCRIPTOR | proto::DISCOVER_TYPE_CONFIG => { + let _ = table.slot(f.param); + } + _ => {} + } + } + + + fn remove(&self) { + self.shutting_down.store(true, Relaxed); + self.release_firmware_mapping(); + trusted::unregister(); + self.unregister_fv_kernel(); + + let dev = self.bio_dev.lock().take(); + if dev.is_some() { + bio::release(&mut self.bio_session.lock()); + sensor::power(false); + sensor::unregister_driver(); + } + drop(dev); + + self.rng_shutdown.store(true, Relaxed); + self.control_wq.notify_all(); + self.sbio_wq.notify_all(); + self.sks_wq.notify_all(); + self.scrd_wq.notify_all(); + if let Some(mut handle) = self.rng.lock().take() { + handle.unregister(); + } + + *self.mbox.lock() = None; + + // SAFETY: all four pointers refer to pinned work fields in `self`. + // Shutdown blocks requeueing, and the mailbox can no longer add work. + unsafe { + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.rx_work)).cast()); + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.enrol_work)).cast()); + sep_cancel_work_sync(Work::raw_get(core::ptr::addr_of!(self.verify_work)).cast()); + sep_cancel_delayed_work_sync( + DelayedWork::raw_as_work(core::ptr::addr_of!(self.settle_work)).cast(), + ); + } + + let _ = self.store.lock().take(); + let _ = self.host_store.lock().take(); + + for slot in [ + &self.ool_xarm, + &self.ool_sbio, + &self.ool_sks, + &self.ool_scrd, + ] { + if let Some(buffers) = slot.lock().take() { + if buffers.registered { + core::mem::forget(buffers); + } + } + } + + if let Some(ring) = self.dma_ring.lock().take() { + if self.sbio_ready.load(Relaxed) { + core::mem::forget(ring); + } + } + + let mut guard = self.shmem.lock(); + if let Some(buf) = guard.take() { + if self.registered.load(Relaxed) { + core::mem::forget(buf); + } else { + drop(buf); + } + } + } +} + +/// # Safety +/// +/// Each is called only by `bio_shim.c`, with the `*const SepData` handed to +/// `BioChardev::register`, which stays valid for the whole registered window. +unsafe extern "C" fn bio_open_trampoline(ctx: *mut c_void) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + match this.bio_open() { + Ok(()) => 0, + Err(e) => e.to_errno(), + } +} + +/// # Safety +unsafe extern "C" fn bio_release_trampoline(ctx: *mut c_void) { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + this.bio_release(); +} + +/// # Safety +unsafe extern "C" fn bio_ioctl_trampoline(ctx: *mut c_void, cmd: c_uint, arg: c_ulong) -> c_long { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + let handled = match this.bio_ioctl(cmd, arg) { + Ok(h) => h, + Err(e) => return e.to_errno() as c_long, + }; + + if handled.start_verify { + // SAFETY: as for `start_enrol` below: same pointer, same guarantee. + let borrow = unsafe { kernel::sync::ArcBorrow::::from_raw(ctx.cast::()) }; + SepData::queue_verify(borrow.into()); + } + + if handled.start_enrol { + // SAFETY: `ctx` is the pointer the driver's own `Arc` was made + // from and that `Arc` outlives every callback; see the contract on the + // registration. `ArcBorrow` is what turns that guarantee into an owned + // reference with the refcount incremented rather than a second owner of + // the same count. + let borrow = unsafe { kernel::sync::ArcBorrow::::from_raw(ctx.cast::()) }; + SepData::queue_enrolment(borrow.into()); + } + handled.ret as c_long +} + +/// # Safety +unsafe extern "C" fn bio_ready_trampoline(ctx: *mut c_void) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + c_int::from(this.bio_ready()) +} + +/// # Safety +/// +/// Only ever called by the hwrng core, through `hwrng_shim.c`. `ctx` is the +/// `*const SepData` handed to `HwRngHandle::register`, which stays valid for +/// the whole registered window (see the ordering note in `remove()`), and +/// `data` is valid for `max` bytes. +unsafe extern "C" fn hwrng_read_trampoline( + ctx: *mut c_void, + data: *mut c_void, + max: usize, + wait: bool, +) -> c_int { + // SAFETY: per the contract above. + let this = unsafe { &*ctx.cast::() }; + this.hwrng_fill(data.cast::(), max, wait) +} + +impl MailCallback for SepData { + type Data = Arc; + + fn recv_message(data: ::Borrowed<'_>, msg: Message) { + if data.shutting_down.load(Relaxed) { + return; + } + if !data.rx.push(msg) && data.rx.dropped() == 1 { + dev_err!( + data.dev, + "receive ring full; dropping messages (first was msg0 {:#018x})\n", + msg.msg0 + ); + } + + let this: Arc = data.into(); + let _ = workqueue::system().enqueue::, 0>(this); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } + if this.drain() > 0 { + SepData::arm_settle(&this); + } + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } + SepData::settle_tick(&this); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } + this.run_enrolment(); + } +} + +impl WorkItem for SepData { + type Pointer = Arc; + + fn run(this: Arc) { + if this.shutting_down.load(Relaxed) { + return; + } + this.run_verify(); + } +} + + +struct SepDriver(Arc); + +const OF_TABLE: kernel::device_id::IdArray = + kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); + +impl platform::Driver for SepDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _info: Option<&()>, + ) -> impl PinInit { + let dev: &device::Device = pdev.as_ref(); + if *module_parameters::provision_keybag.value() != 0 + && *module_parameters::xart_writes.value() == 0 + { + dev_err!(dev, "provision_keybag=1 requires xart_writes=1\n"); + return Err(EINVAL); + } + let sep_node = dt::DtNode::of_device(dev).ok_or(ENODEV)?; + + if dt::registration_already_sent(&sep_node) { + dev_err!( + dev, + "the shared-memory registration was already sent on this boot (marker property present). It is one-shot per AP reset and cannot be repeated or withdrawn: reboot to attach again. Not probing.\n" + ); + return Err(EBUSY); + } + + let data = SepData::new(pdev)?; + + *data.mbox.lock() = Some(Mailbox::new_byname(dev, c"mbox", data.clone())?); + + data.attach(&sep_node)?; + + if let Err(e) = data.register_fv_kernel() { + dev_err!(dev, "could not register the FileVault kernel API: {:?}\n", e); + } + + if let Err(e) = trusted::register(data.clone()) { + dev_warn!(data.dev, "trusted-keys: registration failed ({:?})\n", e); + } + + if data.registered.load(Relaxed) { + SepData::arm_settle(&data); + } + + Ok(SepDriver(data)) + } +} + +impl Drop for SepDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + + +#[pin_data] +struct SepModule { + // field-init order enables the DT nodes before the driver registers; a failure here aborts module init + _dt: (), + + #[pin] + _driver: driver::Registration>, +} + +impl kernel::InPlaceModule for SepModule { + fn init(module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + // The warm-attach target enables the SEP and its DART with a runtime + // changeset; the cold-boot target describes them statically (its DART + // is enabled before probe), so no changeset runs there. + _dt: { + if let Ok(p) = profile::detect() { + if matches!(p.bootstrap, profile::Bootstrap::WarmRegister) { + dt::enable_sep_and_dart()?; + } + } + }, + + _driver <- driver::Registration::new( + ::NAME, + module, + ), + }) + } +} + +module! { + type: SepModule, + name: "apple_sep", + description: "Apple SEP coprocessor: warm attach, endpoint discovery and tag-correlated control endpoint", + license: "Dual MIT/GPL", + params: { + xart_writes: u8 { + default: 0, + description: "Allow writes to the validated shared xART store", + }, + xart_start_sector: u64 { + default: 0, + description: "Explicit 512-byte start sector of the xART gigalocker extent within the iBoot system container. Zero (default) locates the gigalocker automatically in the container by its root records; a non-zero value overrides the search. Either way the driver opens the container directly", + }, + provision_keybag: u8 { + default: 0, + description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + }, + os_uuid_hi: u64 { + default: 0, + description: "High 64 bits of an explicit xART OS UUID", + }, + os_uuid_lo: u64 { + default: 0, + description: "Low 64 bits of an explicit xART OS UUID", + }, + }, } diff --git a/drivers/soc/apple/sha_shim.c b/drivers/soc/apple/sha_shim.c new file mode 100644 index 00000000000000..2e823df021dde3 --- /dev/null +++ b/drivers/soc/apple/sha_shim.c @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* SHA-256 shim: hashes two caller-chosen segments; no protocol logic here. */ + +#include +#include +#include + +#include "shim.h" + +/* + * Hashes `a` then `b`, writes 32 bytes to `out`. Allocates a transform per call + * and may sleep. Returns 0 or a negative errno; `out` invalid on failure. + */ +int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, + unsigned char *out) +{ + struct crypto_shash *tfm; + int rc; + + tfm = crypto_alloc_shash("sha256", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + { + SHASH_DESC_ON_STACK(desc, tfm); + + desc->tfm = tfm; + + rc = crypto_shash_init(desc); + if (!rc && alen) + rc = crypto_shash_update(desc, a, alen); + if (!rc && blen) + rc = crypto_shash_update(desc, b, blen); + if (!rc) + rc = crypto_shash_final(desc, out); + + /* The digest is over a request that may contain a secret. */ + shash_desc_zero(desc); + } + + crypto_free_shash(tfm); + return rc; +} diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h new file mode 100644 index 00000000000000..2797f679f2e5e5 --- /dev/null +++ b/drivers/soc/apple/shim.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — private declarations for the C shim. + * + * Glue between the Rust driver and kernel interfaces whose ABI is awkward or + * unsafe to restate in Rust: `struct hwrng`, whose layout moves with + * CONFIG_LOCKDEP; the file interface, whose O_* flags are macro-derived and so + * absent from the Rust bindings; and the crypto API, whose shash descriptor is + * sized by a macro at its call site. + * + * No protocol logic lives in the shim. Included only by the shim translation + * units. + */ + +#ifndef SEP_SHIM_H +#define SEP_SHIM_H + +#include + +struct apple_sep_fv_new_file_key; + +void sep_cancel_work_sync(void *work); +void sep_cancel_delayed_work_sync(void *work); + +/* -- hwrng_shim.c ------------------------------------------------------- */ + +void *sep_hwrng_alloc(void); +void sep_hwrng_free(void *mem); +int sep_hwrng_register(void *mem, const char *name, + unsigned short quality, void *ctx, + int (*read)(void *ctx, void *data, size_t max, + bool wait)); +void sep_hwrng_unregister(void *mem); + +/* -- store_shim.c ------------------------------------------------------- */ + +void *sep_store_open(const char *path); +void *sep_store_open_trunc(const char *path); +void *sep_store_open_ro(const char *path); +void *sep_store_open_block(const char *path, int writable); +void sep_store_close(void *handle); +long long sep_store_size(void *handle); +long sep_store_read(void *handle, long long off, void *buf, size_t len); +long sep_store_write(void *handle, long long off, const void *buf, + size_t len); +int sep_store_sync(void *handle); + +/* -- sha_shim.c --------------------------------------------------------- */ + +int sep_sha256(const void *a, size_t alen, const void *b, size_t blen, + unsigned char *out); + +/* -- crypto_shim.c ------------------------------------------------------ */ + +int sep_random_bytes(void *buf, size_t len); + +/* HMAC-SHA256 over one message, 32 bytes out. */ +int sep_hmac_sha256(const void *key, size_t keylen, + const void *data, size_t datalen, u8 *out); + +/* + * AES-256-GCM in place over `buf` = `aadlen` bytes of AAD followed by the + * payload. Returns -EBADMSG on auth failure, which the caller must treat as + * fatal. + */ +int sep_gcm(int encrypt, const void *key, size_t keylen, + const void *iv, size_t ivlen, size_t aadlen, + void *buf, size_t buflen, size_t datalen); + +/* -- fv_shim.c -------------------------------------------------------- */ + +struct apple_sep_fv_key; + +struct sep_fv_ops { + int (*unwrap_media_key)(void *context, const u8 *wrapped, + size_t wrapped_len, u32 protection_class, + struct apple_sep_fv_key *key); + int (*unwrap_volume_key)(void *context, const u8 *secret, + size_t secret_len, const u8 *unlock_record, + size_t unlock_record_len, const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key); + int (*load_class_keys)(void *context, const u8 volume_uuid[16], + const u8 *secret, + size_t secret_len, const u8 *unlock_record, + size_t unlock_record_len, const u8 *volume_key, + size_t volume_key_len); + int (*unload_class_keys)(void *context, const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len); + int (*unwrap_file_key)(void *context, const u8 volume_uuid[16], + u32 protection_class, const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key); + int (*new_file_key)(void *context, const u8 volume_uuid[16], + u32 protection_class, u64 crypto_id, + u16 key_revision, + struct apple_sep_fv_new_file_key *key); +}; + +int sep_fv_register_v2(void *context, const struct sep_fv_ops *ops); +void sep_fv_unregister_v2(void *context); + +/* -- p256_shim.c ------------------------------------------------------- */ + +/* + * ECIES sender key agreement for the ref-key (op 0x22) SE-seal, with a fresh + * ephemeral P-256 keypair. `peer_pub_be` is the recipient's 64-byte public + * point (0x04 prefix stripped), `eph_pub_be_out` receives the 64-byte ephemeral + * public point, `shared_be_out` the 32-byte shared secret. All big-endian. + */ +int sep_p256_sender(const void *peer_pub_be, void *eph_pub_be_out, + void *shared_be_out); + +/* -- sensor_shim.c ------------------------------------------------------ */ + +int sep_sensor_register(void); +void sep_sensor_unregister(void); +int sep_sensor_bound(void); +int sep_sensor_power_line(void); +int sep_sensor_cs_timing_mode(void); +int sep_sensor_power_cycle(void); +int sep_sensor_power_source(void); +int sep_sensor_power(int on); +int sep_sensor_xfer(const void *tx, void *rx, size_t len); +int sep_sensor_xfer_tx(const void *tx, size_t len); +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len); + +/* + * Optional data-ready interrupt for interrupt-driven capture. Set up once while + * the sensor is idle (never toggled per capture), then armed and waited on to + * wake the capture loop the instant a frame is ready instead of polling. + */ +int sep_sensor_irq_setup(void); +void sep_sensor_irq_teardown(void); +int sep_sensor_irq_available(void); +void sep_sensor_irq_arm(void); +int sep_sensor_irq_wait(unsigned int timeout_ms); + +/* -- bio_shim.c --------------------------------------------------------- */ + +void *sep_bio_register(const char *name, unsigned short mode, void *ctx, + int (*f_open)(void *), + void (*f_release)(void *), + long (*f_ioctl)(void *, unsigned int, unsigned long), + int (*f_ready)(void *)); +void sep_bio_unregister(void *dev); +void sep_bio_wake(void *dev); +int sep_bio_capable_admin(void); +__u64 sep_bio_monotonic_ns(void); +__u64 sep_bio_boottime_ns(void); + +/* -- trusted_shim.c ---------------------------------------------------- */ + +/* + * The kernel trusted-key source (keyctl). The shim owns the framework structs + * and registration; the sealing policy is in Rust (trusted.rs), reached through + * the callbacks passed to sep_tk_register(). The payload struct is only + * forward-declared here since these prototypes take it by pointer; trusted_shim.c + * pulls in for the definition. + */ +struct trusted_key_payload; + +typedef int (*sep_tk_init_fn)(void); +typedef int (*sep_tk_seal_fn)(struct trusted_key_payload *p, char *datablob); +typedef int (*sep_tk_unseal_fn)(struct trusted_key_payload *p, char *datablob); +typedef int (*sep_tk_random_fn)(unsigned char *key, size_t key_len); +typedef void (*sep_tk_exit_fn)(void); + +int sep_tk_register(sep_tk_init_fn init, sep_tk_seal_fn seal, + sep_tk_unseal_fn unseal, sep_tk_random_fn random, + sep_tk_exit_fn exit); +void sep_tk_unregister(void); + +int sep_tk_register_key_type(void); +void sep_tk_unregister_key_type(void); + +size_t sep_tk_max_key_size(void); +size_t sep_tk_max_blob_size(void); + +unsigned char *sep_tk_key_ptr(struct trusted_key_payload *p); +unsigned int sep_tk_key_len(const struct trusted_key_payload *p); +void sep_tk_set_key_len(struct trusted_key_payload *p, unsigned int n); + +unsigned char *sep_tk_blob_ptr(struct trusted_key_payload *p); +unsigned int sep_tk_blob_len(const struct trusted_key_payload *p); +void sep_tk_set_blob_len(struct trusted_key_payload *p, unsigned int n); + +#endif /* SEP_SHIM_H */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs new file mode 100644 index 00000000000000..f02a9b86e32ef1 --- /dev/null +++ b/drivers/soc/apple/shim.rs @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Rust side of `store_shim.c`: the backing-store file. + +use kernel::prelude::*; + +extern "C" { + fn sep_store_open(path: *const c_char) -> *mut c_void; + fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; + fn sep_store_open_ro(path: *const c_char) -> *mut c_void; + fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; + fn sep_store_close(handle: *mut c_void); + fn sep_store_size(handle: *mut c_void) -> i64; + fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) -> c_long; + fn sep_store_write(handle: *mut c_void, off: i64, buf: *const c_void, len: usize) -> c_long; + fn sep_store_sync(handle: *mut c_void) -> c_int; + fn sep_random_bytes(buf: *mut c_void, len: usize) -> c_int; +} + +pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { + // SAFETY: `buf.as_mut_ptr()` is valid for writes of `buf.len()` bytes for + // the duration of the call, and `sep_random_bytes` writes exactly that many + // bytes (via `get_random_bytes`) and retains no reference to the buffer. + let ret = unsafe { sep_random_bytes(buf.as_mut_ptr().cast(), buf.len()) }; + kernel::error::to_result(ret) +} + +/// Backing-store file handle. +/// +/// # Invariants +/// `handle` is a live `struct file *` from `sep_store_open`. +pub(crate) struct StoreFile { + handle: *mut c_void, +} + +// SAFETY: a `struct file *` has no thread affinity; every access goes through +// the C shim, and the driver keeps this behind a mutex. +unsafe impl Send for StoreFile {} + +fn result_of(ret: c_long) -> Result { + if ret < 0 { + Err(Error::from_errno(ret as c_int)) + } else { + Ok(ret as usize) + } +} + +impl StoreFile { + pub(crate) fn open(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_trunc(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_readonly(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. + let handle = unsafe { sep_store_open_ro(path.as_char_ptr()) }; + if handle.is_null() { + return Err(ENOENT); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn open_block(path: &CStr, writable: bool) -> Result { + // SAFETY: `path` is NUL-terminated; the shim accepts only a block + // device whose global read-only state matches `writable`. + let handle = unsafe { sep_store_open_block(path.as_char_ptr(), c_int::from(writable)) }; + if handle.is_null() { + return Err(ENODEV); + } + Ok(StoreFile { handle }) + } + + pub(crate) fn size(&self) -> Result { + // SAFETY: `handle` is live per the type invariant. + let n = unsafe { sep_store_size(self.handle) }; + if n < 0 { + return Err(EIO); + } + Ok(n as u64) + } + + pub(crate) fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + let mut done = 0usize; + while done < buf.len() { + // SAFETY: `handle` is live; the pointer and length describe the + // remaining tail of `buf`, which we own mutably. + let n = result_of(unsafe { + sep_store_read( + self.handle, + (off + done as u64) as i64, + buf[done..].as_mut_ptr().cast::(), + buf.len() - done, + ) + })?; + if n == 0 { + return Err(EIO); + } + done += n; + } + Ok(()) + } + + pub(crate) fn read_block_exact(&self, off: u64, buf: &mut [u8]) -> Result<()> { + // SAFETY: `handle` remains live and `buf` is writable for its length. + let n = result_of(unsafe { + sep_store_read( + self.handle, + off as i64, + buf.as_mut_ptr().cast::(), + buf.len(), + ) + })?; + if n != buf.len() { + return Err(EIO); + } + Ok(()) + } + + pub(crate) fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { + let mut done = 0usize; + while done < buf.len() { + // SAFETY: as above, for a shared borrow. + let n = result_of(unsafe { + sep_store_write( + self.handle, + (off + done as u64) as i64, + buf[done..].as_ptr().cast::(), + buf.len() - done, + ) + })?; + if n == 0 { + return Err(EIO); + } + done += n; + } + Ok(()) + } + + pub(crate) fn write_block_exact(&self, off: u64, buf: &[u8]) -> Result<()> { + // SAFETY: `handle` remains live and `buf` is readable for its length. + let n = result_of(unsafe { + sep_store_write( + self.handle, + off as i64, + buf.as_ptr().cast::(), + buf.len(), + ) + })?; + if n != buf.len() { + return Err(EIO); + } + Ok(()) + } + + pub(crate) fn sync(&self) -> Result<()> { + // SAFETY: `handle` is live per the type invariant. + kernel::error::to_result(unsafe { sep_store_sync(self.handle) }) + } +} + +impl Drop for StoreFile { + fn drop(&mut self) { + // SAFETY: `handle` is live per the type invariant and is not used again. + unsafe { sep_store_close(self.handle) }; + } +} + +extern "C" { + fn sep_bio_register( + name: *const c_char, + mode: u16, + ctx: *mut c_void, + f_open: Option c_int>, + f_release: Option, + f_ioctl: Option c_long>, + f_ready: Option c_int>, + ) -> *mut c_void; + fn sep_bio_unregister(dev: *mut c_void); + fn sep_bio_wake(dev: *mut c_void); + fn sep_bio_capable_admin() -> c_int; + fn sep_bio_monotonic_ns() -> u64; + fn sep_bio_boottime_ns() -> u64; +} + +pub(crate) fn capable_admin() -> bool { + // SAFETY: no preconditions; reads the current task's credentials. + unsafe { sep_bio_capable_admin() != 0 } +} + +pub(crate) fn monotonic_ns() -> u64 { + // SAFETY: no preconditions. + unsafe { sep_bio_monotonic_ns() } +} + +/// `CLOCK_BOOTTIME`; differs from monotonic by suspend time — how "tokens don't +/// survive a suspend" is detected. +pub(crate) fn boottime_ns() -> u64 { + // SAFETY: no preconditions. + unsafe { sep_bio_boottime_ns() } +} + +/// The registered character device. +/// +/// # Invariants +/// `dev` is a live registration from `sep_bio_register`. +pub(crate) struct BioChardev { + dev: *mut c_void, +} + +// SAFETY: the registration has no thread affinity; every access goes through +// the C shim, which locks internally. +unsafe impl Send for BioChardev {} + +impl BioChardev { + /// Registers `/dev/`. + /// + /// # Safety + /// `ctx` must stay valid and safe to pass to the four callbacks until this + /// is dropped; the callbacks may run on any task at any time. + pub(crate) unsafe fn register( + name: &'static CStr, + mode: u16, + ctx: *mut c_void, + f_open: unsafe extern "C" fn(*mut c_void) -> c_int, + f_release: unsafe extern "C" fn(*mut c_void), + f_ioctl: unsafe extern "C" fn(*mut c_void, c_uint, c_ulong) -> c_long, + f_ready: unsafe extern "C" fn(*mut c_void) -> c_int, + ) -> Result { + // SAFETY: `name` is a static NUL-terminated string that outlives the + // registration, and the caller guarantees the same of `ctx`. + let dev = unsafe { + sep_bio_register( + name.as_char_ptr(), + mode, + ctx, + Some(f_open), + Some(f_release), + Some(f_ioctl), + Some(f_ready), + ) + }; + if dev.is_null() { + return Err(ENODEV); + } + Ok(BioChardev { dev }) + } + + pub(crate) fn wake(&self) { + // SAFETY: `dev` is live per the type invariant. + unsafe { sep_bio_wake(self.dev) }; + } +} + +impl Drop for BioChardev { + fn drop(&mut self) { + // SAFETY: `dev` is live per the type invariant and not used again. + // Deregistration waits for open files, so no callback runs once this returns. + unsafe { sep_bio_unregister(self.dev) }; + } +} diff --git a/drivers/soc/apple/shmem.rs b/drivers/soc/apple/shmem.rs new file mode 100644 index 00000000000000..0c930939d3a0fd --- /dev/null +++ b/drivers/soc/apple/shmem.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use kernel::device; +use kernel::dma; +use kernel::platform; +use kernel::prelude::*; + +const ENTRY_SIZE: usize = 16; +const ENTRY_OFF_FOURCC: usize = 0; +const ENTRY_OFF_SIZE: usize = 4; +const ENTRY_OFF_OFFSET: usize = 8; + +// Buffer-format constant, not PAGE_SIZE despite the coincidence. +const PAYLOAD_BASE: usize = 0x4000; +const PAYLOAD_ALIGN: usize = 0x4000; + +const CINP_MIN_SIZE: usize = 0x8000; + +const CINP_PAYLOAD: [u8; 1] = [0]; + +// Wire byte order, not byte-reversed; llun is the terminator spelling. The +// first item's fourcc is per-SoC and comes from the platform profile. +const FOURCC_OPLA: &[u8; 4] = b"OPLA"; +const FOURCC_IPIS: &[u8; 4] = b"IPIS"; +const FOURCC_TERM: &[u8; 4] = b"llun"; + +const PROP_LOCAL_POLICY: &CStr = c"local-policy-manifest"; +const PROP_IBOOT: &CStr = c"iboot-manifest"; + +pub(crate) type ShMem = dma::Coherent<[u8]>; + +#[derive(Clone, Copy)] +pub(crate) struct Region { + pub(crate) offset: usize, + // Entry size field = allocation size, not payload length. + pub(crate) size: usize, + pub(crate) payload_len: usize, +} + +impl Region { + fn place(offset: usize, payload_len: usize, min_size: usize) -> Region { + let mut size = align_up(payload_len + 4, PAYLOAD_ALIGN); + if size < min_size { + size = min_size; + } + Region { + offset, + size, + payload_len, + } + } + + fn end(&self) -> usize { + self.offset + self.size + } +} + +pub(crate) struct Shmem { + pub(crate) buf: ShMem, +} + +const fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +fn write_at(buf: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { + let end = off.checked_add(src.len()).ok_or(EINVAL)?; + if end > buf.len() { + return Err(EINVAL); + } + // SAFETY: runs in probe before the SEP is told the buffer exists, and probe + // is the only writer, so nothing else accesses it. + unsafe { + buf.as_mut()[off..end].copy_from_slice(src); + } + Ok(()) +} + +fn write_entry( + buf: &mut ShMem, + index: usize, + fourcc: &[u8; 4], + size: usize, + offset: usize, +) -> Result<()> { + let base = index * ENTRY_SIZE; + if base + ENTRY_SIZE > PAYLOAD_BASE { + return Err(EINVAL); + } + write_at(buf, base + ENTRY_OFF_FOURCC, fourcc)?; + let size32: u32 = size.try_into().map_err(|_| EINVAL)?; + write_at(buf, base + ENTRY_OFF_SIZE, &size32.to_le_bytes())?; + // u32 offset at byte 8 (written as u64 with zero pad), never byte 12. + let offset64: u64 = offset.try_into().map_err(|_| EINVAL)?; + write_at(buf, base + ENTRY_OFF_OFFSET, &offset64.to_le_bytes())?; + Ok(()) +} + +fn read_blob(dev: &device::Device, name: &CStr) -> Result> { + let fwnode = dev.fwnode().ok_or(EIO)?; + let len = fwnode.property_count_elem::(name)?; + if len == 0 { + return Err(ENODATA); + } + fwnode + .property_read_array_vec::(name, len)? + .required_by(dev) +} + +fn read_manifest(dev: &device::Device, name: &CStr) -> Result> { + match read_blob(dev, name) { + Ok(v) => Ok(v), + Err(e) => { + dev_err!( + dev, + "refusing to attach: device-tree property '{}' missing or unreadable ({:?}); a CINP-only table would burn the one-shot registration and fault\n", + name, + e + ); + Err(e) + } + } +} + +fn verify_layout( + dev: &device::Device, + cinp: &Region, + opla: &Region, + ipis: &Region, + used: usize, + capacity: usize, + first_item: &[u8; 4], +) -> Result<()> { + let regions = [ + (first_item, cinp), + (FOURCC_OPLA, opla), + (FOURCC_IPIS, ipis), + ]; + + for (name, r) in regions { + let bad = r.offset < PAYLOAD_BASE + || r.offset % PAYLOAD_ALIGN != 0 + || r.size % PAYLOAD_ALIGN != 0 + || r.size < r.payload_len + 4 + || r.end() > capacity; + if bad { + dev_err!( + dev, + "refusing to attach: item '{}' region is malformed (offset 0x{:x}, size 0x{:x}, payload {} bytes)\n", + core::str::from_utf8(name).unwrap_or("????"), + r.offset, + r.size, + r.payload_len + ); + return Err(EINVAL); + } + } + + if cinp.offset != PAYLOAD_BASE + || opla.offset != cinp.end() + || ipis.offset != opla.end() + || used != ipis.end() + { + dev_err!( + dev, + "refusing to attach: regions do not tile from 0x{:x} (CINP 0x{:x}+0x{:x}, OPLA 0x{:x}+0x{:x}, IPIS 0x{:x}+0x{:x}, used 0x{:x})\n", + PAYLOAD_BASE, + cinp.offset, + cinp.size, + opla.offset, + opla.size, + ipis.offset, + ipis.size, + used + ); + return Err(EINVAL); + } + + if cinp.size < CINP_MIN_SIZE { + return Err(EINVAL); + } + + if 4 * ENTRY_SIZE > PAYLOAD_BASE || used > capacity { + return Err(ENOSPC); + } + + Ok(()) +} + +pub(crate) fn build( + pdev: &platform::Device, + capacity: usize, + first_item: &[u8; 4], +) -> Result { + let dev: &device::Device = pdev.as_ref(); + + // Read manifests before allocating: a CINP-only registration would burn the one-shot and fault. + let opla_blob = read_manifest(dev, PROP_LOCAL_POLICY)?; + let ipis_blob = read_manifest(dev, PROP_IBOOT)?; + + let cinp = Region::place(PAYLOAD_BASE, CINP_PAYLOAD.len(), CINP_MIN_SIZE); + let opla = Region::place(cinp.end(), opla_blob.len(), 0); + let ipis = Region::place(opla.end(), ipis_blob.len(), 0); + let used = ipis.end(); + + verify_layout(dev, &cinp, &opla, &ipis, used, capacity, first_item)?; + + let mut buf = dma::Coherent::::zeroed_slice(dev, capacity, GFP_KERNEL)?; + + // Payloads before entries: a failure leaves an all-zero table, not a valid-looking one. + write_at( + &mut buf, + cinp.offset, + &(cinp.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, cinp.offset + 4, &CINP_PAYLOAD)?; + + write_at( + &mut buf, + opla.offset, + &(opla.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, opla.offset + 4, &opla_blob)?; + + write_at( + &mut buf, + ipis.offset, + &(ipis.payload_len as u32).to_le_bytes(), + )?; + write_at(&mut buf, ipis.offset + 4, &ipis_blob)?; + + write_entry(&mut buf, 0, first_item, cinp.size, cinp.offset)?; + write_entry(&mut buf, 1, FOURCC_OPLA, opla.size, opla.offset)?; + write_entry(&mut buf, 2, FOURCC_IPIS, ipis.size, ipis.offset)?; + write_entry(&mut buf, 3, FOURCC_TERM, 0, 0)?; + + Ok(Shmem { buf }) +} diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs new file mode 100644 index 00000000000000..03caee8f0767b2 --- /dev/null +++ b/drivers/soc/apple/sks.rs @@ -0,0 +1,1193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj +//! SEP key store (SKS, endpoint `0x12`): key-bag and key-management request/reply +//! framing, DER-imaged request builders, and lock-state control. + +use super::*; +use crate::proto::*; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +struct KeybagCreateIntent<'a> { + dev: &'a device::Device, + slot: keybag::Slot, + sent: bool, +} + +impl<'a> KeybagCreateIntent<'a> { + fn new(dev: &'a device::Device, slot: keybag::Slot) -> Option { + if let Err(e) = keybag::write_intent(slot) { + dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); + return None; + } + Some(Self { + dev, + slot, + sent: false, + }) + } + + fn sending(&mut self) { + self.sent = true; + } +} + +impl Drop for KeybagCreateIntent<'_> { + fn drop(&mut self) { + if !self.sent { + if let Err(e) = keybag::mark_refused(self.slot) { + dev_err!( + self.dev, + "sks: cannot mark unsent keybag create retryable: {:?}\n", + e + ); + } + } + } +} + +impl SepData { + pub(crate) fn on_sks(&self, msg: Message) { + let r = crate::sks::decode_sks(&msg); + let mut probe = self.sks_probe.lock(); + if probe.active { + if probe.captured.len() < SKS_MAX_CAPTURE { + let _ = probe.captured.push(msg, GFP_KERNEL); + } + drop(probe); + self.sks_wq.notify_all(); + } else { + let matched = probe + .abandoned + .iter() + .flatten() + .find(|a| a.selector == r.selector && a.seq == r.seq) + .copied(); + drop(probe); + if let Some(a) = matched { + let _ = self.sks_zero_buffers(); + self.sks_wedged.store(0, Relaxed); + dev_warn!( + self.dev, + "sks: late answer to {} after {} ms; wedge lifted\n", + a.label, + a.waited_ms + ); + } + } + } + + fn sks_note_abandoned(&self, selector: u8, seq: u8, label: &'static CStr, waited_ms: u64) { + let mut probe = self.sks_probe.lock(); + let slot = probe.abandoned_next; + probe.abandoned[slot] = Some(Abandoned { + selector, + seq, + label, + waited_ms, + }); + probe.abandoned_next = (slot + 1) % SKS_MAX_ABANDONED; + } + + fn sks_read_buffer(&self, inbound: bool, len: usize) -> Result> { + let guard = self.ool_sks.lock(); + let pair = guard.as_ref().ok_or(ENODEV)?; + let src = if inbound { + &pair.inbound + } else { + &pair.outbound + }; + if len > pair.allocated { + return Err(EINVAL); + } + let mut out = KVec::new(); + // SAFETY: the enclave writes this buffer then signals the mailbox, so it is + // quiescent when we read it; the driver is the only other accessor. + out.extend_from_slice(unsafe { &src.as_ref()[..len] }, GFP_KERNEL)?; + Ok(out) + } + + fn sks_zero_buffers(&self) -> Result<()> { + let mut guard = self.ool_sks.lock(); + let pair: &mut Option = &mut guard; + let buffers = pair.as_mut().ok_or(ENODEV)?; + // SAFETY: the enclave touches these only between a request word and its + // reply, and none is outstanding here; the driver is the only other accessor. + unsafe { + buffers.inbound.as_mut()[..buffers.allocated].fill(0); + buffers.outbound.as_mut()[..buffers.allocated].fill(0); + } + Ok(()) + } + + pub(crate) fn ool_registered(&self, slot: &Mutex>) -> bool { + slot.lock().as_ref().is_some_and(|b| b.registered) + } + + pub(crate) fn sks_next_seq(&self) -> crate::sks::Sequence { + let n = self.sks_seq.load(Relaxed); + self.sks_seq.store(n.wrapping_add(1), Relaxed); + crate::sks::Sequence::from_counter(n as u8) + } + + fn sks_exchange( + &self, + label: &'static CStr, + msg: Message, + img: &image::RequestImage, + ) -> Option { + self.sks_exchange_patient(label, msg, img, 0, true) + } + + fn sks_exchange_patient( + &self, + label: &'static CStr, + msg: Message, + img: &image::RequestImage, + floor_ms: time::Msecs, + condemn: bool, + ) -> Option { + let _exchange = self.sks_exchange_lock.lock(); + if !self.ool_registered(&self.ool_sks) { + return None; + } + + if self.sks_wedged.load(Relaxed) != 0 { + return None; + } + + if self.sks_zero_buffers().is_err() { + return None; + } + if self.ool_write(&self.ool_sks, 0, img.as_slice()).is_err() { + return None; + } + + let sized_ms = sks_timeout_for(img.len()); + let timeout_ms = if sized_ms < floor_ms { + floor_ms + } else { + sized_ms + }; + self.sks_arm(); + if self.send(msg).is_err() { + self.sks_disarm(); + let _ = self.sks_zero_buffers(); + return None; + } + let started_ns = crate::shim::boottime_ns(); + + let sent = crate::sks::decode_sks(&msg); + let mut examined = 0usize; + let mut set_aside = 0u32; + let correlating = loop { + self.sks_wait(timeout_ms, examined + 1); + + let next = { + let probe = self.sks_probe.lock(); + let m = probe.captured.get(examined).copied(); + if m.is_some() { + examined += 1; + } + m + }; + + let Some(candidate) = next else { break None }; + + let decoded = crate::sks::decode_sks(&candidate); + if decoded.seq == sent.seq && decoded.selector == sent.selector { + break Some(candidate); + } + + set_aside = set_aside.saturating_add(1); + if set_aside >= SKS_MAX_SET_ASIDE { + break None; + } + }; + + let leftover = self.sks_disarm(); + let correlating = match correlating { + Some(m) => Some(m), + None => leftover.iter().skip(examined).copied().find(|m| { + let d = crate::sks::decode_sks(m); + d.seq == sent.seq && d.selector == sent.selector + }), + }; + + let Some(raw) = correlating else { + let waited_ms = crate::shim::boottime_ns().saturating_sub(started_ns) / 1_000_000; + self.sks_note_abandoned(sent.selector, sent.seq, label, waited_ms); + if condemn { + self.sks_wedged.store(1, Relaxed); + } + dev_err!( + self.dev, + "sks: {} got no reply after {} ms\n", + label, + waited_ms + ); + return None; + }; + + let reply = crate::sks::decode_sks(&raw); + // reply.status is signed here on (e.g. -13, not 0xf3). + + let mut response = Secret::empty(); + if reply.response_size > 0 { + if let Ok(bytes) = self.sks_read_buffer(false, reply.response_size as usize) { + response = Secret(bytes); + } + } + + // The correlated reply means the enclave has finished with both OOL buffers. + let _ = self.sks_zero_buffers(); + + Some(SksOutcome { reply, response }) + } + + pub(crate) fn sks_report_response<'a>( + &self, + _label: &CStr, + out: &'a SksOutcome, + ) -> Option<&'a [u8]> { + if out.response.is_empty() { + return None; + } + + let parsed = match image::parse_response(&out.response) { + Ok(p) => p, + Err(_) => { + return None; + } + }; + Some(parsed.body) + } + + pub(crate) fn sks_timestamp_us(&self) -> u64 { + crate::shim::boottime_ns() / 1000 + } + + pub(crate) fn sks_image_len(&self, img: &image::RequestImage) -> Result { + let declared_in = sks_declared_sizes().0; + if img.len() > declared_in { + return Err(ENOSPC); + } + crate::sks::ImageLen::of(img.len()).ok_or(EINVAL) + } + + fn sks_seal(&self, op: &crate::sks::SksOp, body: &image::Body) -> Result { + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), body)?; + let len = self.sks_image_len(&img)?; + Ok(SksRequest { + name: op.name(), + msg: crate::sks::encode_sks_read(op, self.sks_next_seq(), len), + img, + }) + } + + /// `0x4d` get capabilities. + fn sks_req_get_capabilities(&self) -> Result { + let op = crate::sks::sks_get_capabilities(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(1)?; + body.put_blob(&[])?; + self.sks_seal(&op, &body) + } + + /// `0x0d` designate. + fn sks_req_designate(&self, d: &crate::sks::Designation, secret: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_DESIGNATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(d.source().value())?; + body.put_i32(d.user().special_handle().value())?; + body.put_blob(secret)?; + // Flags is a u64, not u32; a u32 leaves the body short -> enclave answers -13. + body.put_u64(crate::sks::SKS_DESIGNATE_FLAGS)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_designate(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_DESIGNATE_NAME, + msg, + img, + }) + } + + /// `0x05` unload the source handle. + pub(crate) fn sks_req_unload_keybag( + &self, + handle: crate::sks::KeyBagHandle, + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_unload_keybag(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_UNLOAD_NAME, + msg, + img, + }) + } + + /// `0x06` UUID read, against the special handle. + pub(crate) fn sks_req_copy_uuid_special( + &self, + special: crate::sks::SpecialHandle, + ) -> Result { + let op = crate::sks::sks_copy_keybag_uuid(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + self.sks_seal(&op, &body) + } + + pub(crate) fn sks_req_unlock_special( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + healthy: Healthy, + ) -> Result { + let Healthy(()) = healthy; + let mut body = image::Body::new(); + body.put_u32(SKS_LOCK_STATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_i32(LockState::Unlocked.wire())?; + body.put_blob(secret)?; + body.put_u64(SKS_LOCK_STATE_FLAGS)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = + crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOCK_STATE_NAME, + msg, + img, + }) + } + + /// `0x06` copy key-bag UUID. + fn sks_req_copy_keybag_uuid(&self, handle: crate::sks::KeyBagHandle) -> Result { + let op = crate::sks::sks_copy_keybag_uuid(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + fn sks_req_copy_keybag(&self, handle: crate::sks::KeyBagHandle) -> Result { + let op = crate::sks::sks_copy_keybag(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + /// `0x02` copy the designated biometric identity bag. + fn sks_req_copy_keybag_special(&self, handle: crate::sks::SpecialHandle) -> Result { + let op = crate::sks::sks_copy_keybag(); + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + self.sks_seal(&op, &body) + } + + /// `0x03` load key bag. + fn sks_req_load_keybag(&self, wrapped: &[u8]) -> Result { + let mut body = image::Body::new(); + body.put_u32(0)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_blob(wrapped)?; + + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_load(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOAD_NAME, + msg, + img, + }) + } + + fn sks_req_create_identity_keybag( + &self, + secret: &[u8], + uuid: &[u8; crate::sks::SKS_IDENTITY_UUID_LEN], + proof: keybag::NoStoredKeyBag, + ) -> Result { + if proof.slot() != keybag::Slot::Identity { + return Err(EINVAL); + } + // The strict (T8103) and lenient (T6020) enclaves encode the identity + // bag differently; the per-SoC profile carries the correct field values + // so the proven T6020 path is unchanged while T8103 gets the bag type in + // the third word (see profile::KeybagCreate). + let enc = &self.profile.keybag_create; + let mut body = image::Body::new(); + body.put_u32(enc.variant)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_u32(enc.bag_type)?; + body.put_i32(enc.arg)?; + body.put_blob(secret)?; + body.put_blob(&[])?; + body.put_blob(uuid)?; + body.put_blob(&[])?; + body.put_u64(0)?; + body.put_u64(0)?; + body.put_blob(&[])?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_CREATE_NAME, + msg, + img, + }) + } + + pub(crate) fn sks_provision_identity_keybag(&self) -> bool { + let proof = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(_)) => return true, + Ok(keybag::State::Absent(proof)) => proof, + Err(e) => { + dev_err!( + self.dev, + "sks: identity keybag state is ambiguous: {:?}\n", + e + ); + return false; + } + }; + let slot = proof.slot(); + let mut intent = match KeybagCreateIntent::new(&self.dev, slot) { + Some(intent) => intent, + None => return false, + }; + let mut secret_bytes = KVec::new(); + if secret_bytes.resize(SKS_SECRET_LEN, 0, GFP_KERNEL).is_err() + || shim::random_bytes(&mut secret_bytes).is_err() + { + return false; + } + let secret = Secret(secret_bytes); + let mut uuid = [0u8; crate::sks::SKS_IDENTITY_UUID_LEN]; + if shim::random_bytes(&mut uuid).is_err() { + return false; + } + uuid[6] = (uuid[6] & 0x0f) | 0x40; + uuid[8] = (uuid[8] & 0x3f) | 0x80; + + let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { + Ok(request) => request, + Err(e) => { + dev_err!( + self.dev, + "sks: cannot build identity keybag request: {:?}\n", + e + ); + return false; + } + }; + intent.sending(); + let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + return false; + }; + let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + return false; + }; + if out.reply.status != 0 || body.len() < 12 { + dev_err!( + self.dev, + "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", + out.reply.status, + body.len() + ); + return false; + } + let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); + let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); + let Some((_fv_data, end)) = image::read_blob(body, 8) else { + return false; + }; + if variant != self.profile.keybag_create.variant || raw_handle < 0 || end != body.len() + { + dev_err!( + self.dev, + "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", + variant, + raw_handle + ); + return false; + } + let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); + let Some(bag_uuid) = self.sks_read_uuid(handle) else { + return false; + }; + let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + return false; + }; + if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { + dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); + return false; + } + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + dev_info!(self.dev, "sks: identity keybag provisioned\n"); + true + } + + pub(crate) fn sks_send(&self, req: Result) -> Option { + match req { + Ok(r) => self.sks_exchange(r.name, r.msg, &r.img), + Err(_) => None, + } + } + + pub(crate) fn sks_designate_user_keybag( + &self, + handle: crate::sks::KeyBagHandle, + secret: &[u8], + ) { + let Some(user) = crate::sks::DesignateUser::new(SBIO_PROBE_USER_ID) else { + return; + }; + + let designation = crate::sks::Designation::new(handle, user); + + let Some(out) = self.sks_send(self.sks_req_designate(&designation, secret)) else { + dev_warn!( + self.dev, + "sks: DESIGNATE_KEYBAG did not complete; enrolment will refuse\n" + ); + return; + }; + let Some(body) = self.sks_report_response(crate::sks::SKS_DESIGNATE_NAME, &out) else { + return; + }; + + if body.len() != crate::sks::SKS_DESIGNATE_REPLY_LEN { + return; + } + let variant = u32::from_le_bytes([body[0], body[1], body[2], body[3]]); + if variant != crate::sks::SKS_DESIGNATE_VARIANT { + return; + } + + self.keybag_designated.store(true, Relaxed); + + self.sks_remember_enrolment_material(designation.user().special_handle(), secret); + } + + fn sks_remember_enrolment_material(&self, special: crate::sks::SpecialHandle, secret: &[u8]) { + let mut copy = KVec::new(); + if copy.extend_from_slice(secret, GFP_KERNEL).is_err() { + return; + } + *self.enrol_material.lock() = Some(EnrolMaterial { + special, + secret: Secret(copy), + }); + } + + /// `0x04` change lock state (v1). + pub(crate) fn sks_req_change_lock_state( + &self, + handle: crate::sks::KeyBagHandle, + state: LockState, + secret: &[u8], + healthy: Healthy, + ) -> Result { + let Healthy(()) = healthy; + let mut body = image::Body::new(); + body.put_u32(SKS_LOCK_STATE_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(handle.value())?; + body.put_i32(state.wire())?; + body.put_blob(secret)?; + // Trailing u64 goes after the blob (`0x18` puts it before). + body.put_u64(SKS_LOCK_STATE_FLAGS)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = + crate::sks::encode_sks_change_lock_state(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_LOCK_STATE_NAME, + msg, + img, + }) + } + + /// `0x21` verify-secret (variant 1). + pub(crate) fn sks_req_verify_secret( + &self, + special: crate::sks::SpecialHandle, + secret: &[u8], + acm_context: &[u8; crate::scrd::SCRD_ACM_HANDLE_LEN], + ) -> Result { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_VERIFY_SECRET_VARIANT)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_i32(special.value())?; + body.put_blob(secret)?; + body.put_blob(acm_context)?; + body.put_u64(0)?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_verify_secret(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { + name: crate::sks::SKS_VERIFY_SECRET_NAME, + msg, + img, + }) + } + + pub(crate) fn sks_step( + &self, + label: &CStr, + build: impl FnOnce(Healthy) -> Result, + ) -> Option { + let healthy = self.sks_health_check(label)?; + let out = self.sks_send(build(healthy))?; + if out.reply.status != 0 { + return None; + } + Some(out) + } + + pub(crate) fn sks_health_check(&self, why: &CStr) -> Option { + let Some(out) = self.sks_send(self.sks_req_get_capabilities()) else { + dev_err!( + self.dev, + "sks: health check ({}) got no reply; endpoint gone for this boot\n", + why + ); + return None; + }; + if out.reply.status != 0 { + dev_err!( + self.dev, + "sks: health check ({}) returned status {}\n", + why, + out.reply.status + ); + return None; + } + Some(Healthy(())) + } + + pub(crate) fn sks_recover( + &self, + stored: &keybag::StoredKeyBag, + ) -> Option<(crate::sks::KeyBagHandle, [u8; keybag::UUID_LEN])> { + let request = match self.sks_req_load_keybag(stored.wrapped()) { + Ok(request) => request, + Err(e) => { + dev_err!( + self.dev, + "sks: could not build LOAD_KEYBAG request: {:?}\n", + e + ); + return None; + } + }; + let out = self.sks_exchange(request.name, request.msg, &request.img)?; + + dev_info!( + self.dev, + "sks: LOAD_KEYBAG reply mailbox status {}, response size {}, copied {} bytes\n", + out.reply.status, + out.reply.response_size, + out.response.len() + ); + + let body = match self.sks_report_response(crate::sks::SKS_LOAD_NAME, &out) { + Some(body) => body, + None => { + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG reply image was empty or malformed\n" + ); + return None; + } + }; + + if out.reply.status != 0 { + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG mailbox status {}; no handle\n", + out.reply.status + ); + return None; + } + if body.len() != SKS_LOAD_REPLY_LEN { + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG returned {} bytes, expected {}\n", + body.len(), + SKS_LOAD_REPLY_LEN + ); + return None; + } + let status = i32::from_le_bytes([body[0], body[1], body[2], body[3]]); + let handle = i32::from_le_bytes([body[4], body[5], body[6], body[7]]); + if status != 0 || handle < 0 { + dev_warn!( + self.dev, + "sks: LOAD_KEYBAG operation status {}, handle {}\n", + status, + handle + ); + return None; + } + let handle = crate::sks::KeyBagHandle::from_load_reply(handle); + + let Some(uuid) = self.sks_read_uuid(handle) else { + dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + return None; + }; + + if uuid == *stored.uuid() { + Some((handle, uuid)) + } else { + dev_err!( + self.dev, + "sks: load succeeded but bag identity mismatch: stored UUID {}, loaded UUID {}\n", + Hex(stored.uuid()), + Hex(&uuid) + ); + let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + None + } + } + + pub(crate) fn resnapshot_identity_keybag(&self) -> bool { + let special = { + let material = self.enrol_material.lock(); + let Some(material) = material.as_ref() else { + return false; + }; + material.special + }; + + let Some(out) = self.sks_send(self.sks_req_copy_keybag_special(special)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"the designated identity bag") + else { + return false; + }; + + let Some(uuid) = self + .sks_send(self.sks_req_copy_uuid_special(special)) + .and_then(|out| self.sks_uuid_from_reply(&out)) + else { + return false; + }; + + match keybag::replace_wrapped(keybag::Slot::Identity, &wrapped, &uuid) { + Ok(()) => true, + Err(e) => { + dev_err!( + self.dev, + "enrol: could not commit the identity-bag snapshot ({:?}); may not survive reboot\n", + e + ); + false + } + } + } + + fn sks_read_uuid(&self, handle: crate::sks::KeyBagHandle) -> Option<[u8; keybag::UUID_LEN]> { + let out = self.sks_send(self.sks_req_copy_keybag_uuid(handle))?; + self.sks_uuid_from_reply(&out) + } + + pub(crate) fn sks_uuid_from_reply(&self, out: &SksOutcome) -> Option<[u8; keybag::UUID_LEN]> { + let body = self.sks_report_response(c"COPY_KEYBAG_UUID", out)?; + + if out.reply.status != 0 || image::operation_status(body).unwrap_or(-1) != 0 { + return None; + } + let (blob, _) = image::read_blob(body, 4)?; + if blob.len() != keybag::UUID_LEN { + return None; + } + let mut uuid = [0u8; keybag::UUID_LEN]; + uuid.copy_from_slice(blob); + Some(uuid) + } + + pub(crate) fn enable_sks(&self) -> Result<()> { + self.register_ool(&self.ool_sks) + } + + pub(crate) fn sks_ready(&self) -> bool { + if self.ool_registered(&self.ool_sks) { + return true; + } + if !self.endpoint_present(proto::EP_SKS) { + dev_err!( + self.dev, + "sks: endpoint 0x{:02x} (key store) not advertised; {} EPs up (control={} xarm={} sbio={} scrd={}); keybag/ref-key cannot run\n", + proto::EP_SKS, + self.endpoint_count(), + self.endpoint_present(proto::EP_CONTROL) as u8, + self.endpoint_present(proto::EP_XARM) as u8, + self.endpoint_present(proto::EP_SBIO) as u8, + self.endpoint_present(proto::EP_SCRD) as u8, + ); + return false; + } + if let Err(e) = self.enable_sks() { + dev_err!( + self.dev, + "sks: could not register out-of-line buffers ({:?})\n", + e + ); + return false; + } + dev_info!( + self.dev, + "sks: endpoint 0x{:02x} (key store) up; {} EPs; out-of-line buffers registered\n", + proto::EP_SKS, + self.endpoint_count() + ); + true + } + + fn sks_arm(&self) { + let mut probe = self.sks_probe.lock(); + probe.captured.clear(); + probe.active = true; + } + + fn sks_disarm(&self) -> KVec { + let mut probe = self.sks_probe.lock(); + probe.active = false; + core::mem::take(&mut probe.captured) + } + + fn sks_wait(&self, ms: time::Msecs, until: usize) { + let mut remaining = time::msecs_to_jiffies(ms); + let mut guard = self.sks_probe.lock(); + loop { + if guard.captured.len() >= until || remaining == 0 { + return; + } + match self + .sks_wq + .wait_interruptible_timeout(&mut guard, remaining) + { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } +} + +pub(crate) const SKS_REPLY_BIT: u8 = 0x80; + +pub(crate) struct SksOp { + opcode: u8, + name: &'static CStr, +} + +impl SksOp { + pub(crate) fn opcode(&self) -> u8 { + self.opcode + } + pub(crate) fn name(&self) -> &'static CStr { + self.name + } +} + +const OP_SKS_UNWRAP_PFK: u8 = 0x09; +pub(crate) const SKS_UNWRAP_PFK_NAME: &CStr = c"UNWRAP_PFK"; + +const OP_SKS_NEW_PFK: u8 = 0x10; +pub(crate) const SKS_NEW_PFK_NAME: &CStr = c"NEW_PFK"; + +const OP_SKS_UNWRAP_MEDIA_KEY: u8 = 0x32; +pub(crate) const SKS_UNWRAP_MEDIA_KEY_NAME: &CStr = c"UNWRAP_MEDIA_KEY"; + +const OP_SKS_UNWRAP_VEK: u8 = 0x41; +pub(crate) const SKS_UNWRAP_VEK_NAME: &CStr = c"FV_UNWRAP_VEK"; + +const OP_SKS_SET_PROTECTION: u8 = 0x47; +pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; + +const OP_SKS_GET_BLOB_STATE: u8 = 0x48; +pub(crate) const SKS_GET_BLOB_STATE_NAME: &CStr = c"FV_GET_BLOB_STATE"; + +const OP_SKS_PERFORM_OPERATION: u8 = 0x22; +pub(crate) const SKS_PERFORM_OP_NAME: &CStr = c"PERFORM_OPERATION"; + +pub(crate) fn encode_sks_perform_operation(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_PERFORM_OPERATION, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_unwrap_media_key(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNWRAP_MEDIA_KEY, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_unwrap_vek(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNWRAP_VEK, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_unwrap_pfk(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_UNWRAP_PFK, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_new_pfk(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_NEW_PFK, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_set_protection(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_SET_PROTECTION, seq.value(), len.value()) +} + +pub(crate) fn encode_sks_get_blob_state(seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(OP_SKS_GET_BLOB_STATE, seq.value(), len.value()) +} +const OP_SKS_CAPABILITIES: u8 = 0x4d; +pub(crate) fn sks_get_capabilities() -> SksOp { + SksOp { + opcode: OP_SKS_CAPABILITIES, + name: c"GET_CAPABILITIES", + } +} + +const OP_SKS_COPY_UUID: u8 = 0x06; +pub(crate) fn sks_copy_keybag_uuid() -> SksOp { + SksOp { + opcode: OP_SKS_COPY_UUID, + name: c"COPY_KEYBAG_UUID", + } +} + +const OP_SKS_COPY_KEYBAG: u8 = 0x02; +pub(crate) fn sks_copy_keybag() -> SksOp { + SksOp { + opcode: OP_SKS_COPY_KEYBAG, + name: c"COPY_KEYBAG", + } +} + +const OP_SKS_CREATE_KEYBAG: u8 = 0x01; +pub(crate) const SKS_CREATE_NAME: &CStr = c"CREATE_KEYBAG"; +pub(crate) const SKS_IDENTITY_UUID_LEN: usize = 16; + +const OP_SKS_LOAD_KEYBAG: u8 = 0x03; + +pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; + +const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; + +pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; + +const OP_SKS_VERIFY_SECRET: u8 = 0x21; +pub(crate) const SKS_VERIFY_SECRET_NAME: &CStr = c"VERIFY_SECRET"; +pub(crate) const SKS_VERIFY_SECRET_VARIANT: u32 = 1; + +pub(crate) fn encode_sks_verify_secret(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_VERIFY_SECRET, seq.value(), len.value()); + Some(msg) +} + +pub(crate) const SKS_CLIENT_ID: u64 = u64::from_be_bytes(*b"LINUXSKS"); +static_assert!(SKS_CLIENT_ID == 0x4c49_4e55_5853_4b53); + +#[derive(Clone, Copy)] +pub(crate) struct KeyBagHandle(i32); + +impl KeyBagHandle { + pub(crate) const fn from_create_reply(v: i32) -> KeyBagHandle { + KeyBagHandle(v) + } + + pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { + KeyBagHandle(v) + } + + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +const OP_SKS_DESIGNATE_KEYBAG: u8 = 0x0d; + +pub(crate) const SKS_DESIGNATE_VARIANT: u32 = 1; +static_assert!(SKS_DESIGNATE_VARIANT == 1); + +pub(crate) const SKS_DESIGNATE_USER_MIN: i32 = 10; +static_assert!(SKS_DESIGNATE_USER_MIN > 0); + +#[derive(Clone, Copy)] +pub(crate) struct DesignateUser(i32); + +impl DesignateUser { + pub(crate) fn new(value: i32) -> Option { + if value >= SKS_DESIGNATE_USER_MIN { + Some(DesignateUser(value)) + } else { + None + } + } + + pub(crate) const fn special_handle(&self) -> SpecialHandle { + SpecialHandle(-self.0) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct SpecialHandle(i32); + +impl SpecialHandle { + pub(crate) const fn value(&self) -> i32 { + self.0 + } +} + +pub(crate) const SKS_AUTH_TOKEN_LEN: usize = 16; + +pub(crate) struct Designation { + source: KeyBagHandle, + user: DesignateUser, +} + +impl Designation { + pub(crate) fn new(source: KeyBagHandle, user: DesignateUser) -> Designation { + Designation { source, user } + } + + pub(crate) const fn source(&self) -> KeyBagHandle { + self.source + } + + pub(crate) const fn user(&self) -> DesignateUser { + self.user + } +} + +pub(crate) const SKS_DESIGNATE_REPLY_LEN: usize = 4; + +pub(crate) const SKS_DESIGNATE_FLAGS: u64 = 0; +static_assert!(SKS_DESIGNATE_FLAGS == 0); + +pub(crate) fn encode_sks_designate(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_DESIGNATE_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +pub(crate) const SKS_DESIGNATE_NAME: &CStr = c"DESIGNATE_KEYBAG"; + +const OP_SKS_UNLOAD_KEYBAG: u8 = 0x05; +pub(crate) const SKS_UNLOAD_NAME: &CStr = c"UNLOAD_KEYBAG"; + +pub(crate) fn encode_sks_unload_keybag(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_UNLOAD_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +#[derive(Clone, Copy)] +pub(crate) struct Sequence(u8); + +const SEQ_MIN: u8 = 0x60; + +impl Sequence { + pub(crate) const fn from_counter(n: u8) -> Sequence { + Sequence(SEQ_MIN | (n & !SEQ_MIN)) + } + + pub(crate) const fn value(&self) -> u8 { + self.0 + } +} + +static_assert!(Sequence::from_counter(0).value() != Sequence::from_counter(1).value()); +static_assert!(Sequence::from_counter(0).value() == Sequence::from_counter(32).value()); + +#[derive(Clone, Copy)] +pub(crate) struct ImageLen(u16); + +impl ImageLen { + pub(crate) fn of(len: usize) -> Option { + if len < crate::image::HEADER_WIRE { + return None; + } + match u16::try_from(len) { + Ok(v) => Some(ImageLen(v)), + Err(_) => None, + } + } + + pub(crate) const fn value(&self) -> u16 { + self.0 + } +} + +const fn encode_sks_raw(selector: u8, seq: u8, len: u16) -> Message { + Message { + msg0: (EP_SKS as u64) + | ((selector as u64) << MSG_TAG_SHIFT) + | ((seq as u64) << MSG_TYPE_SHIFT) + | ((len as u64) << 48), + msg1: 0, + } +} + +pub(crate) fn encode_sks_read(op: &SksOp, seq: Sequence, len: ImageLen) -> Message { + encode_sks_raw(op.opcode(), seq.value(), len.value()) +} + +pub(crate) fn encode_sks_load(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_LOAD_KEYBAG, seq.value(), len.value()); + Some(msg) +} + +pub(crate) fn encode_sks_create(seq: Sequence, len: ImageLen) -> Option { + Some(encode_sks_raw( + OP_SKS_CREATE_KEYBAG, + seq.value(), + len.value(), + )) +} + +pub(crate) fn encode_sks_change_lock_state(seq: Sequence, len: ImageLen) -> Option { + let msg = encode_sks_raw(OP_SKS_CHANGE_LOCK_STATE, seq.value(), len.value()); + Some(msg) +} + +pub(crate) struct SksReply { + pub(crate) selector: u8, + pub(crate) seq: u8, + pub(crate) status: i8, + pub(crate) response_size: u16, +} + +pub(crate) fn decode_sks(msg: &Message) -> SksReply { + let b = msg.msg0.to_le_bytes(); + SksReply { + selector: b[1] & !SKS_REPLY_BIT, + seq: b[2], + status: b[3] as i8, + response_size: u16::from_le_bytes([b[6], b[7]]), + } +} diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs new file mode 100644 index 00000000000000..dc37025ce3b8ad --- /dev/null +++ b/drivers/soc/apple/store.rs @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Linux-only state. SEP anti-replay records live in `xart_store`. + +use crate::shim; +use kernel::prelude::*; + +pub(crate) const STORE_PATH: &CStr = c"/var/lib/aurora-sep-host-state.bin"; + +const BLOCK_SIZE: usize = 0x8000; +const BLOCK_COUNT: usize = 72; +pub(crate) const STORE_SIZE: usize = BLOCK_SIZE * BLOCK_COUNT; +const SLOT_COUNT: usize = BLOCK_COUNT - 1; +pub(crate) const MAX_VALUE: usize = BLOCK_SIZE; + +const MAGIC: [u8; 16] = *b"AURORA-SEP-STOR\x01"; +const VERSION: u32 = 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0u8; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + kind: u8, + uuid: [u8; 16], + len: u32, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + kind: 0, + uuid: [0; 16], + len: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.kind == key.kind && self.uuid == key.uuid + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Intent { + None, + Write { slot: u16, kind: u8 }, +} + +const SB_MAGIC: usize = 0; +const SB_VERSION: usize = 16; +const SB_SLOT_COUNT: usize = 20; +const SB_BLOCK_SIZE: usize = 24; +const SB_GENERATION: usize = 28; +const SB_INTENT_KIND: usize = 36; +const SB_INTENT_SLOT: usize = 37; +const SB_INTENT_TYPE: usize = 39; +const SB_SLOT_TABLE: usize = 64; +const SLOT_ENTRY_SIZE: usize = 24; + +const INTENT_NONE: u8 = 0; +const INTENT_WRITE: u8 = 1; + +// Not internally locked; the caller holds a mutex around all access. +pub(crate) struct Store { + file: shim::StoreFile, + slots: [Slot; SLOT_COUNT], + generation: u64, + pub(crate) recovered: bool, + pub(crate) fresh: bool, +} + +fn le32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn le64(buf: &[u8], off: usize) -> u64 { + let mut v = [0u8; 8]; + v.copy_from_slice(&buf[off..off + 8]); + u64::from_le_bytes(v) +} + +impl Store { + pub(crate) fn open() -> Result { + let file = shim::StoreFile::open(STORE_PATH)?; + + let mut store = Store { + file, + slots: [Slot::FREE; SLOT_COUNT], + generation: 0, + recovered: false, + fresh: false, + }; + + let size = store.file.size()?; + if size < STORE_SIZE as u64 { + store.create()?; + return Ok(store); + } + + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + store.file.read_exact(0, &mut sb)?; + + if sb[SB_MAGIC..SB_MAGIC + 16] != MAGIC + || le32(&sb, SB_VERSION) != VERSION + || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT + || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE + { + store.create()?; + return Ok(store); + } + + store.generation = le64(&sb, SB_GENERATION); + for i in 0..SLOT_COUNT { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&sb[off + 2..off + 18]); + store.slots[i] = Slot { + used: sb[off] != 0, + kind: sb[off + 1], + uuid, + len: le32(&sb, off + 18), + }; + } + + // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. + let intent_kind = sb[SB_INTENT_KIND]; + if intent_kind != INTENT_NONE { + store.recovered = true; + let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; + if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + store.slots[slot] = Slot::FREE; + } + store.commit()?; + } + + Ok(store) + } + + fn create(&mut self) -> Result<()> { + self.slots = [Slot::FREE; SLOT_COUNT]; + self.generation = 1; + self.fresh = true; + + let mut zero = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + zero.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + for b in 0..BLOCK_COUNT { + self.file.write_all((b * BLOCK_SIZE) as u64, &zero)?; + } + + self.commit()?; + Ok(()) + } + + fn commit(&mut self) -> Result<()> { + self.write_superblock(Intent::None)?; + self.file.sync() + } + + fn write_superblock(&mut self, intent: Intent) -> Result<()> { + let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; + sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; + + sb[SB_MAGIC..SB_MAGIC + 16].copy_from_slice(&MAGIC); + sb[SB_VERSION..SB_VERSION + 4].copy_from_slice(&VERSION.to_le_bytes()); + sb[SB_SLOT_COUNT..SB_SLOT_COUNT + 4].copy_from_slice(&(SLOT_COUNT as u32).to_le_bytes()); + sb[SB_BLOCK_SIZE..SB_BLOCK_SIZE + 4].copy_from_slice(&(BLOCK_SIZE as u32).to_le_bytes()); + sb[SB_GENERATION..SB_GENERATION + 8].copy_from_slice(&self.generation.to_le_bytes()); + + let (ikind, islot, itype) = match intent { + Intent::None => (INTENT_NONE, 0u16, 0u8), + Intent::Write { slot, kind } => (INTENT_WRITE, slot, kind), + }; + sb[SB_INTENT_KIND] = ikind; + sb[SB_INTENT_SLOT..SB_INTENT_SLOT + 2].copy_from_slice(&islot.to_le_bytes()); + sb[SB_INTENT_TYPE] = itype; + + for (i, slot) in self.slots.iter().enumerate() { + let off = SB_SLOT_TABLE + i * SLOT_ENTRY_SIZE; + sb[off] = u8::from(slot.used); + sb[off + 1] = slot.kind; + sb[off + 2..off + 18].copy_from_slice(&slot.uuid); + sb[off + 18..off + 22].copy_from_slice(&slot.len.to_le_bytes()); + } + + self.file.write_all(0, &sb) + } + + fn find(&self, key: &Key) -> Option { + self.slots.iter().position(|s| s.matches(key)) + } + + fn find_free(&self) -> Option { + self.slots.iter().position(|s| !s.used) + } + + fn block_offset(slot: usize) -> u64 { + ((slot + 1) * BLOCK_SIZE) as u64 + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let len = self.slots[idx].len as usize; + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.resize(len, 0, GFP_KERNEL)?; + if len > 0 { + self.file.read_exact(Self::block_offset(idx), &mut value)?; + } + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !(0xf0..=0xf5).contains(&key.kind) { + return Err(EINVAL); + } + if value.len() > MAX_VALUE { + return Err(ENOSPC); + } + + let old = self.find(key); + let fresh = self.find_free().ok_or(ENOSPC)?; + + // 1. Record the intent, durably, before touching any data block. + self.write_superblock(Intent::Write { + slot: fresh as u16, + kind: key.kind, + })?; + self.file.sync()?; + + // 2. Fill the new block and make it durable. + if !value.is_empty() { + self.file.write_all(Self::block_offset(fresh), value)?; + } + self.file.sync()?; + + // 3. Switch the slot table, release the old block, clear the intent. + self.slots[fresh] = Slot { + used: true, + kind: key.kind, + uuid: key.uuid, + len: value.len() as u32, + }; + if let Some(old) = old { + self.slots[old] = Slot::FREE; + } + self.generation = self.generation.wrapping_add(1); + self.commit() + } +} + +pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { + let mut crc: u16 = 0xffff; + let mut i = 0; + while i < data.len() { + crc ^= (data[i] as u16) << 8; + let mut bit = 0; + while bit < 8 { + crc = if crc & 0x8000 != 0 { + (crc << 1) ^ 0x1021 + } else { + crc << 1 + }; + bit += 1; + } + i += 1; + } + crc +} + +// Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. +static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c new file mode 100644 index 00000000000000..b067c39372fa76 --- /dev/null +++ b/drivers/soc/apple/store_shim.c @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Dj */ +/* + * Backing-store file access. In C because the O_* flags are macro-derived and + * absent from the Rust bindings. + * + * These files are the driver's own. Opens run under kernel credentials so + * access never depends on which task triggered the operation: a keyctl(2) call + * from an unprivileged process reaches unseal in that process's context, and + * without this the driver could not read its own root-only store. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "shim.h" + +static struct file *open_as_kernel(const char *path, int flags, umode_t mode) +{ + const struct cred *old; + struct cred *kern; + struct file *f; + struct path root; + + kern = prepare_kernel_cred(&init_task); + if (!kern) + return ERR_PTR(-ENOMEM); + + task_lock(&init_task); + get_fs_root(init_task.fs, &root); + task_unlock(&init_task); + + old = override_creds(kern); + f = file_open_root(&root, path, flags, mode); + put_cred(revert_creds(old)); + path_put(&root); + + return f; +} + +/* Opens the store, creating if absent. Mode 0600: the file is the driver's own. */ +void *sep_store_open(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_LARGEFILE, 0600); + + return IS_ERR(f) ? NULL : f; +} + +/* + * Opens truncated to empty, for callers that rewrite the whole file each time, + * so a shorter record cannot leave stale trailing bytes from a larger one. + */ +void *sep_store_open_trunc(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + + return IS_ERR(f) ? NULL : f; +} + +/* + * Opens an existing file read-only, NULL if absent. O_RDONLY and no O_CREAT so + * the seed cannot be created, truncated or written by mistake. + */ +void *sep_store_open_ro(const char *path) +{ + struct file *f = open_as_kernel(path, O_RDONLY | O_LARGEFILE, 0); + + return IS_ERR(f) ? NULL : f; +} + +void *sep_store_open_block(const char *path, int writable) +{ + struct file *f = open_as_kernel(path, + writable ? O_RDWR | O_LARGEFILE : O_RDONLY | O_LARGEFILE, + 0); + + if (IS_ERR(f)) + return NULL; + if (!S_ISBLK(file_inode(f)->i_mode) || + bdev_read_only(file_bdev(f)) == !!writable) { + filp_close(f, NULL); + return NULL; + } + return f; +} + +void sep_store_close(void *handle) +{ + if (handle) + filp_close((struct file *)handle, NULL); +} + +long long sep_store_size(void *handle) +{ + struct file *f = handle; + + if (S_ISBLK(file_inode(f)->i_mode)) + return bdev_nr_bytes(file_bdev(f)); + return i_size_read(file_inode(f)); +} + +long sep_store_read(void *handle, long long off, void *buf, size_t len) +{ + struct file *f = handle; + loff_t pos = off; + + return kernel_read(f, buf, len, &pos); +} + +long sep_store_write(void *handle, long long off, const void *buf, + size_t len) +{ + struct file *f = handle; + loff_t pos = off; + + return kernel_write(f, buf, len, &pos); +} + +int sep_store_sync(void *handle) +{ + return vfs_fsync((struct file *)handle, 0); +} diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs new file mode 100644 index 00000000000000..1f77f451625760 --- /dev/null +++ b/drivers/soc/apple/transfer.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use kernel::prelude::*; + +pub(crate) const MARKER_FIRST: u8 = 0xFC; +pub(crate) const MARKER_NEXT: u8 = 0xFD; +/// Requests the peer's next packet and acks the final one; same marker for both. +pub(crate) const MARKER_REQUEST: u8 = 0xFE; +pub(crate) const MARKER_ERROR: u8 = 0xFF; + +// A tag below MARKER_FIRST is not a marker; sound only while every marker >= it. +static_assert!(MARKER_NEXT >= MARKER_FIRST); +static_assert!(MARKER_REQUEST >= MARKER_FIRST); + +#[derive(Clone, Copy)] +pub(crate) enum DeviceStatus { + /// `0xffffffff` is a real refusal (signed −1), not the absence of an answer. + Answered(u32), + ReportedWithoutStatus, + NotAnswered, + BufferNeverWritten, +} + +impl DeviceStatus { + pub(crate) fn answered(&self) -> Option { + match self { + DeviceStatus::Answered(err) => Some(*err), + DeviceStatus::ReportedWithoutStatus + | DeviceStatus::NotAnswered + | DeviceStatus::BufferNeverWritten => None, + } + } + + pub(crate) fn is_ok(&self) -> bool { + matches!(self, DeviceStatus::Answered(0)) + } +} + +impl kernel::fmt::Display for DeviceStatus { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + match self { + DeviceStatus::Answered(err) => write!(f, "0x{:x}", err), + DeviceStatus::ReportedWithoutStatus => write!( + f, + "an error report from the device carrying no status word (the device DID answer)" + ), + DeviceStatus::NotAnswered => write!( + f, + "NONE — no message arrived at all (host sentinel, not an enclave value)" + ), + DeviceStatus::BufferNeverWritten => write!( + f, + "NONE — a payload notification arrived but the outbound buffer was never written (a race with the enclave's DMA, not a silent enclave)" + ), + } + } +} + +pub(crate) const HEADER_WORDS: usize = 7; +pub(crate) const HEADER_LEN: usize = HEADER_WORDS * 4; + +const VERSION: u32 = 1; + +pub(crate) const MAX_TRANSACTION: u32 = 0x4B000; + +#[derive(Clone, Copy)] +pub(crate) struct Packet { + pub(crate) version: u32, + pub(crate) total: u32, + pub(crate) offset: u32, + pub(crate) flags: u32, + pub(crate) err: u32, + pub(crate) opcode: u32, + pub(crate) chunk: u32, +} + +fn le32(buf: &[u8], word: usize) -> u32 { + let o = word * 4; + u32::from_le_bytes([buf[o], buf[o + 1], buf[o + 2], buf[o + 3]]) +} + +impl Packet { + pub(crate) fn decode(buf: &[u8]) -> Result { + if buf.len() < HEADER_LEN { + return Err(EINVAL); + } + Ok(Packet { + version: le32(buf, 0), + total: le32(buf, 1), + offset: le32(buf, 2), + flags: le32(buf, 3), + err: le32(buf, 4), + opcode: le32(buf, 5), + chunk: le32(buf, 6), + }) + } + + pub(crate) fn encode(&self, buf: &mut [u8]) -> Result<()> { + if buf.len() < HEADER_LEN { + return Err(EINVAL); + } + for (word, value) in [ + self.version, + self.total, + self.offset, + self.flags, + self.err, + self.opcode, + self.chunk, + ] + .into_iter() + .enumerate() + { + buf[word * 4..word * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + } + + pub(crate) fn request_next(opcode: u32, received: u32, total: u32) -> Packet { + Packet { + version: VERSION, + total, + offset: received, + flags: 0, + err: 0, + opcode, + chunk: 0, + } + } +} + +struct Active { + opcode: u32, + total: u32, + payload: KVec, + seq: u16, +} + +pub(crate) struct Completed { + pub(crate) status: DeviceStatus, + pub(crate) payload: KVec, +} + +pub(crate) struct Continuation { + opcode: u32, + received: u32, + total: u32, + seq: u16, +} + +impl Continuation { + pub(crate) fn opcode(&self) -> u32 { + self.opcode + } + pub(crate) fn seq(&self) -> u16 { + self.seq + } + pub(crate) fn packet(&self) -> Packet { + Packet::request_next(self.opcode, self.received, self.total) + } +} + +pub(crate) enum Progress { + NeedMore(Continuation), + Complete, + /// A stray chunk, dropped without touching transfer state — distinct from + /// `Failed`, which kills the transfer. + Ignored, + Grant, + Notification, + Failed, +} + +pub(crate) struct Reassembly { + active: Option, + done: Option, + sending: Option, + grants: u32, +} + +impl Reassembly { + pub(crate) fn new() -> Reassembly { + Reassembly { + active: None, + done: None, + sending: None, + grants: 0, + } + } + + pub(crate) fn begin(&mut self, opcode: u32) -> Result<()> { + if self.active.is_some() { + return Err(EBUSY); + } + self.done = None; + self.active = Some(Active { + opcode, + total: 0, + payload: KVec::new(), + seq: 0, + }); + Ok(()) + } + + pub(crate) fn begin_send(&mut self, opcode: u32) { + self.sending = Some(opcode); + self.grants = 0; + } + + pub(crate) fn end_send(&mut self) { + self.sending = None; + self.grants = 0; + } + + pub(crate) fn take_grant(&mut self) -> bool { + if self.grants > 0 { + self.grants -= 1; + return true; + } + false + } + + pub(crate) fn take_done(&mut self) -> Option { + self.done.take() + } + + pub(crate) fn abort(&mut self) { + self.abort_with(DeviceStatus::NotAnswered); + } + + pub(crate) fn abort_with(&mut self, status: DeviceStatus) { + if self.active.take().is_some() { + self.done = Some(Completed { + status, + payload: KVec::new(), + }); + } + } + + fn fail(&mut self) { + self.abort(); + } + + pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { + if marker < MARKER_FIRST { + return Progress::Notification; + } + + // 0xFE is flow control for the request being sent, answered from `sending`. + if marker == MARKER_REQUEST { + return match self.sending { + Some(_) => { + // Counted under the caller's lock, closing the race with the waiter. + self.grants = self.grants.saturating_add(1); + Progress::Grant + } + None => Progress::Ignored, + }; + } + + let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { + return Progress::Ignored; + }; + + // Checked before the opcode test below, deliberately. + if marker == MARKER_ERROR { + let status = if packet.err != 0 { + DeviceStatus::Answered(packet.err) + } else { + DeviceStatus::ReportedWithoutStatus + }; + self.finish(status, KVec::new()); + return Progress::Complete; + } + + // The device echoes the opcode on a data chunk. + if packet.opcode != active_opcode { + return Progress::Ignored; + } + + if packet.version != VERSION { + self.fail(); + return Progress::Failed; + } + if packet.chunk as usize != payload.len() { + self.fail(); + return Progress::Failed; + } + if packet.total > MAX_TRANSACTION { + self.fail(); + return Progress::Failed; + } + + match marker { + MARKER_FIRST => { + let Some(active) = self.active.as_ref() else { + return Progress::Ignored; + }; + if !active.payload.is_empty() { + self.fail(); + return Progress::Failed; + } + if packet.offset != 0 { + self.fail(); + return Progress::Failed; + } + if let Some(active) = self.active.as_mut() { + active.total = packet.total; + } + } + MARKER_NEXT => { + let Some(active) = self.active.as_ref() else { + return Progress::Ignored; + }; + if packet.offset as usize != active.payload.len() { + self.fail(); + return Progress::Failed; + } + if packet.total != active.total { + self.fail(); + return Progress::Failed; + } + } + _ => { + self.fail(); + return Progress::Failed; + } + } + + let Some(active) = self.active.as_mut() else { + return Progress::Ignored; + }; + + if packet.err != 0 { + let status = DeviceStatus::Answered(packet.err); + let collected = core::mem::take(&mut active.payload); + self.finish(status, collected); + return Progress::Complete; + } + + if active + .payload + .extend_from_slice(payload, GFP_KERNEL) + .is_err() + { + self.fail(); + return Progress::Failed; + } + + let received = active.payload.len() as u32; + let total = active.total; + + if received >= total { + let collected = core::mem::take(&mut active.payload); + self.finish(DeviceStatus::Answered(0), collected); + Progress::Complete + } else { + active.seq = active.seq.wrapping_add(1); + Progress::NeedMore(Continuation { + opcode: active_opcode, + received, + total, + seq: active.seq, + }) + } + } + + fn finish(&mut self, status: DeviceStatus, payload: KVec) { + self.active = None; + self.done = Some(Completed { status, payload }); + } +} diff --git a/drivers/soc/apple/trusted.rs b/drivers/soc/apple/trusted.rs new file mode 100644 index 00000000000000..6162bc8af9d3c6 --- /dev/null +++ b/drivers/soc/apple/trusted.rs @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! A Linux trusted-key source backed by the SEP reference-key seal. +//! +//! Seal is a host-side ECIES encrypt to the machine ref-key's public point (no +//! enclave round trip). Unseal asks the enclave to ECIES-decrypt (op 0x22 +//! `"oecd"`) with the in-enclave private, which needs the login key bag reloaded +//! and unlocked first. The ref-key lifecycle lives on [`SepData`]; this is the +//! keyring glue. The config-dependent payload ABI is in `trusted_shim.c`. + +use core::sync::atomic::{AtomicPtr, Ordering}; + +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::types::ForeignOwnable; + +use crate::{Secret, SepData}; + +/// One opaque answer for every unseal rejection: telling a wrong blob, a foreign +/// machine's blob and a tampered blob apart would be an oracle. No name in the +/// Rust error set, hence the literal. +const EBADMSG: c_int = -74; + +/// The framework dispatches the ops with no context pointer, so `SepData` is +/// reached through this global. Holds the [`ForeignOwnable::into_foreign`] +/// pointer while registered; `null` otherwise, and the cmpxchg once-guard. +static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +extern "C" { + fn sep_tk_register( + init: unsafe extern "C" fn() -> c_int, + seal: unsafe extern "C" fn(*mut c_void, *mut c_char) -> c_int, + unseal: unsafe extern "C" fn(*mut c_void, *mut c_char) -> c_int, + random: unsafe extern "C" fn(*mut u8, usize) -> c_int, + exit: unsafe extern "C" fn(), + ) -> c_int; + fn sep_tk_unregister(); + + fn sep_tk_register_key_type() -> c_int; + fn sep_tk_unregister_key_type(); + + fn sep_tk_max_key_size() -> usize; + fn sep_tk_max_blob_size() -> usize; + + fn sep_tk_key_ptr(p: *mut c_void) -> *mut u8; + fn sep_tk_key_len(p: *const c_void) -> c_uint; + fn sep_tk_set_key_len(p: *mut c_void, n: c_uint); + + fn sep_tk_blob_ptr(p: *mut c_void) -> *mut u8; + fn sep_tk_blob_len(p: *const c_void) -> c_uint; + fn sep_tk_set_blob_len(p: *mut c_void, n: c_uint); +} + +/// Soundness rests on [`unregister`] resetting the framework's static calls +/// before it reclaims the `Arc`: no op runs after that, and while one runs the +/// pointer is live. +fn with_sep(f: impl FnOnce(&SepData) -> T) -> Option { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + // SAFETY: produced by `into_foreign` in `register`, reclaimed only by + // `unregister` after ops stop dispatching, so live for this borrow. + let borrow = unsafe { as ForeignOwnable>::borrow(ptr) }; + Some(f(&borrow)) +} + +/// # Safety +/// Called by the key-type core as the `init` op; takes no arguments. +unsafe extern "C" fn tk_init() -> c_int { + let rc = with_sep(|sep| { + dev_info!( + sep.dev, + "trusted-keys: registering \"trusted\" key type backed by the SEP ref-key seal\n" + ); + + // SAFETY: no preconditions; -EEXIST if already registered. + let krc = unsafe { sep_tk_register_key_type() }; + if krc != 0 { + dev_err!( + sep.dev, + "trusted-keys: register_key_type failed ({}); registration aborted\n", + krc + ); + } + krc + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// `datablob` options (keyhandle/hash/policy) are ignored: no migration or +/// policy binding, so a key is bound only to the machine ref-key. +/// +/// # Safety +/// Called by the framework with a live `struct trusted_key_payload *p`. +unsafe extern "C" fn tk_seal(p: *mut c_void, _datablob: *mut c_char) -> c_int { + let rc = with_sep(|sep| { + // SAFETY: `p` is a live payload; key_len <= MAX_KEY_SIZE bytes at p->key. + let key_len = unsafe { sep_tk_key_len(p.cast_const()) } as usize; + // SAFETY: `p` is a live payload. + let key_ptr = unsafe { sep_tk_key_ptr(p) }; + // SAFETY: `key_ptr`/`key_len` describe `p->key`; read-only. + let key = unsafe { core::slice::from_raw_parts(key_ptr.cast_const(), key_len) }; + + let blob = match sep.refkey_seal_trusted(key) { + Ok(blob) => blob, + Err(e) => { + dev_warn!( + sep.dev, + "trusted-keys: seal failed ({:?}); the machine ref-key may not be established yet\n", + e + ); + return e.to_errno(); + } + }; + + // SAFETY: no preconditions. + let max_blob = unsafe { sep_tk_max_blob_size() }; + if blob.len() > max_blob { + dev_warn!( + sep.dev, + "trusted-keys: sealed blob {} byte(s) exceeds MAX_BLOB_SIZE {} byte(s); refusing\n", + blob.len(), + max_blob + ); + return E2BIG.to_errno(); + } + + // SAFETY: `p` is a live payload. + let dst = unsafe { sep_tk_blob_ptr(p) }; + // SAFETY: `dst` is `p->blob` (MAX_BLOB_SIZE bytes), blob.len() <= max_blob, + // no overlap. + unsafe { core::ptr::copy_nonoverlapping(blob.as_ptr(), dst, blob.len()) }; + // SAFETY: `p` is live. + unsafe { sep_tk_set_blob_len(p, blob.len() as c_uint) }; + + // Lengths only, never the key or blob bytes. + dev_info!( + sep.dev, + "trusted-keys: sealed {}-byte key into {}-byte enclave blob\n", + key_len, + blob.len() + ); + 0 + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// # Safety +/// Called by the framework with a live `struct trusted_key_payload *p`. +unsafe extern "C" fn tk_unseal(p: *mut c_void, _datablob: *mut c_char) -> c_int { + let rc = with_sep(|sep| { + // SAFETY: `p` is a live payload; blob_len <= MAX_BLOB_SIZE bytes at p->blob. + let blob_len = unsafe { sep_tk_blob_len(p.cast_const()) } as usize; + // SAFETY: `p` is a live payload. + let blob_ptr = unsafe { sep_tk_blob_ptr(p) }; + // SAFETY: `blob_ptr`/`blob_len` describe `p->blob`; read-only. + let blob = unsafe { core::slice::from_raw_parts(blob_ptr.cast_const(), blob_len) }; + + // Every failure past here is one opaque EBADMSG. + let Ok(plain) = sep.refkey_unseal_trusted(blob) else { + return EBADMSG; + }; + let plain = Secret(plain); + + // SAFETY: no preconditions. + let max_key = unsafe { sep_tk_max_key_size() }; + if plain.is_empty() || plain.len() > max_key { + return EBADMSG; + } + + // SAFETY: `p` is a live payload. + let dst = unsafe { sep_tk_key_ptr(p) }; + // SAFETY: `dst` is `p->key` (MAX_KEY_SIZE + 1 bytes), plain.len() <= max_key, + // no overlap. + unsafe { core::ptr::copy_nonoverlapping(plain.as_ptr(), dst, plain.len()) }; + // SAFETY: `p` is live. + unsafe { sep_tk_set_key_len(p, plain.len() as c_uint) }; + + dev_info!( + sep.dev, + "trusted-keys: unsealed {}-byte enclave blob to {}-byte key\n", + blob_len, + plain.len() + ); + 0 + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// Returns the byte count on success (not 0) per the framework contract, so the +/// caller's `ret != key_len` check passes. +/// +/// # Safety +/// Called by the framework with `key` writable for `key_len` bytes. +unsafe extern "C" fn tk_get_random(key: *mut u8, key_len: usize) -> c_int { + let rc = with_sep(|sep| { + if key_len == 0 { + return 0; + } + // SAFETY: the framework guarantees `key` is writable for `key_len` bytes. + let buf = unsafe { core::slice::from_raw_parts_mut(key, key_len) }; + match sep.sep_random(buf) { + Ok(()) => key_len as c_int, + Err(e) => e.to_errno(), + } + }); + rc.unwrap_or_else(|| ENODEV.to_errno()) +} + +/// # Safety +/// Called by the framework at most once per successful `init()`. +unsafe extern "C" fn tk_exit() { + // SAFETY: no preconditions; the key type was registered by `tk_init`. + unsafe { sep_tk_unregister_key_type() }; +} + +/// Idempotent: the cmpxchg claims the global slot once. On any failure the `Arc` +/// is reclaimed and the slot left clear, so a later call can retry. +pub(crate) fn register(sep: Arc) -> Result<()> { + let ptr = sep.into_foreign(); + + if SEP + .compare_exchange( + core::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_err() + { + // SAFETY: `ptr` was just produced by `into_foreign` and never published. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + return Err(EBUSY); + } + + // SAFETY: the shim stores these callbacks and calls + // `register_trusted_key_source`; valid for the module's lifetime. + let rc = unsafe { sep_tk_register(tk_init, tk_seal, tk_unseal, tk_get_random, tk_exit) }; + if rc != 0 { + // Nothing was wired, so no op can dispatch; clear the slot and reclaim. + let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !raw.is_null() { + // SAFETY: `raw` is the pointer we published; the ops are not wired. + drop(unsafe { as ForeignOwnable>::from_foreign(raw) }); + } + return Err(Error::from_errno(rc)); + } + Ok(()) +} + +/// Safe to call even if [`register`] never ran. Resets the framework's static +/// calls and runs `exit()` before reclaiming the `Arc`, so no op is in flight +/// against freed data once this returns. +pub(crate) fn unregister() { + if SEP.load(Ordering::Acquire).is_null() { + return; + } + // SAFETY: no preconditions; matched with the `sep_tk_register` above. + unsafe { sep_tk_unregister() }; + + let ptr = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); + if !ptr.is_null() { + // SAFETY: `ptr` came from `into_foreign` in `register`, and no op runs now. + drop(unsafe { as ForeignOwnable>::from_foreign(ptr) }); + } +} diff --git a/drivers/soc/apple/trusted_shim.c b/drivers/soc/apple/trusted_shim.c new file mode 100644 index 00000000000000..ed34794031f476 --- /dev/null +++ b/drivers/soc/apple/trusted_shim.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Dj */ +/* + * Apple SEP driver — trusted-key source shim (device-rooted sealing wired to + * the kernel trusted-key framework). + * + * An ABI the Rust bindings do not cover. 's struct + * trusted_key_payload is sized by MAX_KEY_SIZE / MAX_BLOB_SIZE and embeds an + * rcu_head, so its layout moves with the kernel config; and + * register_trusted_key_source / unregister_trusted_key_source, key_type_trusted + * and register_key_type() are C symbols absent from the Rust bindings. + * + * No sealing policy lives here: this file owns the struct handling and the + * registration only. init/seal/unseal/get_random/exit live in trusted.rs and + * are reached through the callbacks stored below. No key material is touched. + */ + +#include +#include +#include +#include + +#include "shim.h" + +/* + * register_trusted_key_source / unregister_trusted_key_source are declared by + * and exported by the trusted-key core. Not re-declared + * here: restating them risks a conflicting type (unregister returns void, not + * int). register validates src->{name,ops->{init,seal,unseal}}, honours + * trusted.source=, returns -EBUSY if another source is active, calls + * src->ops->init(), then wires the static calls; unregister resets those static + * calls, runs src->ops->exit(), and clears the active source. + */ + +/* Rust callbacks (trusted.rs); the sep_tk_*_fn types are declared in shim.h. */ +static sep_tk_init_fn rs_init; +static sep_tk_seal_fn rs_seal; +static sep_tk_unseal_fn rs_unseal; +static sep_tk_random_fn rs_random; +static sep_tk_exit_fn rs_exit; + +/* + * Trampolines for the framework's ops table, each forwarding to the stored Rust + * callback. Keeping these pointers in C means no Rust symbol is exported by + * name and the ops table stays a plain static initialiser. + */ +static int tk_init(void) +{ + return rs_init ? rs_init() : -ENODEV; +} + +static int tk_seal(struct trusted_key_payload *p, char *datablob) +{ + return rs_seal ? rs_seal(p, datablob) : -ENODEV; +} + +static int tk_unseal(struct trusted_key_payload *p, char *datablob) +{ + return rs_unseal ? rs_unseal(p, datablob) : -ENODEV; +} + +static int tk_get_random(unsigned char *key, size_t key_len) +{ + return rs_random ? rs_random(key, key_len) : -ENODEV; +} + +static void tk_exit(void) +{ + if (rs_exit) + rs_exit(); +} + +/* + * A writable array, not a string literal, so .name assigns cleanly whether the + * framework declares it char * or const char *. + */ +static char sep_tk_name[] = "applesep"; + +static struct trusted_key_ops sep_tk_ops = { + .migratable = 0, + .init = tk_init, + .seal = tk_seal, + .unseal = tk_unseal, + .get_random = tk_get_random, + .exit = tk_exit, +}; + +static struct trusted_key_source sep_tk_source = { + .name = sep_tk_name, + .ops = &sep_tk_ops, +}; + +/* + * Store the callbacks, then register. They must be in place first because + * register_trusted_key_source() calls init() synchronously. + */ +int sep_tk_register(sep_tk_init_fn init, sep_tk_seal_fn seal, + sep_tk_unseal_fn unseal, sep_tk_random_fn random, + sep_tk_exit_fn exit) +{ + rs_init = init; + rs_seal = seal; + rs_unseal = unseal; + rs_random = random; + rs_exit = exit; + return register_trusted_key_source(&sep_tk_source); +} + +void sep_tk_unregister(void) +{ + unregister_trusted_key_source(&sep_tk_source); + rs_init = NULL; + rs_seal = NULL; + rs_unseal = NULL; + rs_random = NULL; + rs_exit = NULL; +} + +/* + * Key-type lifecycle, owned by the source's init()/exit(), as the in-tree + * trusted-key sources (dcp, pkwm, tpm) do. + */ +int sep_tk_register_key_type(void) +{ + return register_key_type(&key_type_trusted); +} + +void sep_tk_unregister_key_type(void) +{ + unregister_key_type(&key_type_trusted); +} + +/* Compile-time facts the Rust side bounds-checks against. */ +size_t sep_tk_max_key_size(void) +{ + return MAX_KEY_SIZE; +} + +size_t sep_tk_max_blob_size(void) +{ + return MAX_BLOB_SIZE; +} + +/* + * Payload accessors. The struct layout tracks MAX_KEY_SIZE / MAX_BLOB_SIZE, so + * these wrappers keep the offsets in C where the header defines them. + */ +unsigned char *sep_tk_key_ptr(struct trusted_key_payload *p) +{ + return p->key; +} + +unsigned int sep_tk_key_len(const struct trusted_key_payload *p) +{ + return p->key_len; +} + +void sep_tk_set_key_len(struct trusted_key_payload *p, unsigned int n) +{ + p->key_len = n; +} + +unsigned char *sep_tk_blob_ptr(struct trusted_key_payload *p) +{ + return p->blob; +} + +unsigned int sep_tk_blob_len(const struct trusted_key_payload *p) +{ + return p->blob_len; +} + +void sep_tk_set_blob_len(struct trusted_key_payload *p, unsigned int n) +{ + p->blob_len = n; +} diff --git a/drivers/soc/apple/work_shim.c b/drivers/soc/apple/work_shim.c new file mode 100644 index 00000000000000..ad8e147cd6f63f --- /dev/null +++ b/drivers/soc/apple/work_shim.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include + +#include "shim.h" + +void sep_cancel_work_sync(void *work) +{ + cancel_work_sync(work); +} + +void sep_cancel_delayed_work_sync(void *work) +{ + cancel_delayed_work_sync(work); +} diff --git a/drivers/soc/apple/xarm.rs b/drivers/soc/apple/xarm.rs new file mode 100644 index 00000000000000..ae9fe467280207 --- /dev/null +++ b/drivers/soc/apple/xarm.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +use crate::store::crc16_ccitt_false; +use crate::xarm::{XarmReply as Reply, XarmRequest as Request}; +use crate::xart_store::{Key, Store, MAX_VALUE}; +use kernel::prelude::*; +use kernel::soc::apple::mailbox::Message; + +pub(crate) use crate::proto::EP_XARM; + +const OP_ROOT_READ: u8 = 0x00; +const OP_ROOT_WRITE: u8 = 0x01; +const OP_SESSION_READ: u8 = 0x05; +const OP_SESSION_WRITE: u8 = 0x06; +const OP_SESSION_DELETE: u8 = 0x07; +pub(crate) const OP_QUERY_PROTECTED: u8 = 0x0e; +const OP_GET_OS_UUID: u8 = 0x13; +const OP_NOTIFY_DISABLE_FIRST: u8 = 0x1d; +const OP_NOTIFY_DISABLE_LAST: u8 = 0x1f; + +pub(crate) const STATUS_OK: u8 = 0x00; +const STATUS_UNAVAILABLE: u8 = 0x02; +pub(crate) const STATUS_FAILED: u8 = 0x16; + +const ROOT_TYPE_BASE: u8 = 1; +const SESSION_TYPE_BASE: u8 = 3; + +/// Root read prefixes its value with seventeen zero bytes. +const ROOT_READ_PREFIX: usize = 17; + +/// `0x01` rejects when `(length >> 4) >= 0x7ff`. +const ROOT_WRITE_MAX: usize = 0x7fef; + +impl Reply { + fn ok(req: &Request, length: u16) -> Reply { + Reply { + tag: req.tag, + status: STATUS_OK, + length, + args: [0; 3], + } + } + + fn fail(req: &Request, status: u8) -> Reply { + Reply { + tag: req.tag, + status, + length: 0, + args: [0; 3], + } + } +} + +pub(crate) fn is_silent(opcode: u8) -> bool { + (OP_NOTIFY_DISABLE_FIRST..=OP_NOTIFY_DISABLE_LAST).contains(&opcode) +} + +pub(crate) fn needs_buffers(opcode: u8) -> bool { + opcode != OP_QUERY_PROTECTED +} + +fn root_key(args: &[u8; 3]) -> Key { + Key::root(ROOT_TYPE_BASE + (args[0] & 1)) +} + +fn session_type(args: &[u8; 3]) -> u8 { + SESSION_TYPE_BASE + (args[0] & 1) +} + +fn expected_crc(args: &[u8; 3]) -> u16 { + u16::from_le_bytes([args[1], args[2]]) +} + +fn uuid_from(bytes: &[u8]) -> [u8; 16] { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&bytes[..16]); + uuid +} + +pub(crate) struct Serviced { + pub(crate) reply: Reply, + pub(crate) reply_bytes: usize, +} + +pub(crate) fn service( + req: &Request, + outbound: &[u8], + inbound: &mut [u8], + store: &mut Store, + protected_data: bool, + os_uuid: Option<[u8; 16]>, +) -> Serviced { + match req.opcode { + OP_ROOT_READ => { + let key = root_key(&req.args); + let value = match store.read(&key) { + Ok(v) => v, + Err(_) => { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + } + } + }; + + let Some(value) = value else { + return Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }; + }; + let total = ROOT_READ_PREFIX + value.len(); + if total > inbound.len() { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..ROOT_READ_PREFIX].fill(0); + inbound[ROOT_READ_PREFIX..total].copy_from_slice(&value); + + Serviced { + reply: Reply::ok(req, total as u16), + reply_bytes: total, + } + } + + OP_ROOT_WRITE => { + if (req.length as usize) > ROOT_WRITE_MAX || outbound.len() > MAX_VALUE { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + if crc16_ccitt_false(outbound) != expected_crc(&req.args) { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + match store.write(&root_key(&req.args), outbound) { + Ok(()) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_READ => { + if outbound.len() != 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.read(&key) { + Ok(Some(value)) => { + if value.len() > inbound.len() { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..value.len()].copy_from_slice(&value); + Serviced { + reply: Reply::ok(req, value.len() as u16), + reply_bytes: value.len(), + } + } + Ok(None) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_WRITE => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + if crc16_ccitt_false(outbound) != expected_crc(&req.args) { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.write(&key, &outbound[16..]) { + Ok(()) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_SESSION_DELETE => { + if outbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + let key = Key::new(session_type(&req.args), uuid_from(outbound)); + match store.delete(&key) { + Ok(true) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Ok(false) => Serviced { + reply: Reply::ok(req, 0), + reply_bytes: 0, + }, + Err(_) => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } + } + + OP_QUERY_PROTECTED => { + let mut reply = Reply::ok(req, 0); + reply.args[0] = u8::from(protected_data); + Serviced { + reply, + reply_bytes: 0, + } + } + + OP_GET_OS_UUID => match os_uuid { + Some(uuid) => { + if inbound.len() < 16 { + return Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }; + } + inbound[..16].copy_from_slice(&uuid); + // Reply length must echo the request, not the 16 flushed bytes; setting 16 is wrong. + Serviced { + reply: Reply::ok(req, req.length), + reply_bytes: 16, + } + } + None => Serviced { + reply: Reply::fail(req, STATUS_UNAVAILABLE), + reply_bytes: 0, + }, + }, + + _ => Serviced { + reply: Reply::fail(req, STATUS_FAILED), + reply_bytes: 0, + }, + } +} + +pub(crate) struct XarmRequest { + pub(crate) tag: u8, + pub(crate) opcode: u8, + pub(crate) length: u16, + pub(crate) args: [u8; 3], +} + +pub(crate) struct XarmReply { + pub(crate) tag: u8, + pub(crate) status: u8, + pub(crate) length: u16, + pub(crate) args: [u8; 3], +} + +pub(crate) fn decode_xarm(msg: &Message) -> XarmRequest { + let b = msg.msg0.to_le_bytes(); + XarmRequest { + tag: b[1], + opcode: b[2], + length: u16::from_le_bytes([b[3], b[4]]), + args: [b[5], b[6], b[7]], + } +} + +pub(crate) fn encode_xarm_reply(reply: &XarmReply) -> Message { + let len = reply.length.to_le_bytes(); + Message { + msg0: u64::from_le_bytes([ + EP_XARM, + reply.tag, + reply.status, + len[0], + len[1], + reply.args[0], + reply.args[1], + reply.args[2], + ]), + msg1: 0, + } +} + +static_assert!(EP_XARM == 0x13); diff --git a/drivers/soc/apple/xart_store.rs b/drivers/soc/apple/xart_store.rs new file mode 100644 index 00000000000000..ac9e0bf6c51e24 --- /dev/null +++ b/drivers/soc/apple/xart_store.rs @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Dj + +//! Device-wide xART gigalocker record store. +//! +//! The APFS locator exposes the existing `.gl` file as a block device. This +//! module implements the record validation, duplicate repair, lookup and +//! copy-on-write ordering the SEP requires. It never creates storage +//! and it can be opened read-only for safe inspection and bring-up. + +use crate::shim; +use kernel::prelude::*; + +/// The raw-extent owner opens this whole partition (the iBoot system container, +/// where the gigalocker lives) and serves the gigalocker window directly. +pub(crate) const OWNER_PATH: &CStr = c"/dev/disk/by-partlabel/iBootSystemContainer"; + +const BLOCK_SIZE: usize = 0x1000; +const SLOT_SIZE: usize = 0x9000; +const HEADER_SIZE: usize = 0x22; +const DELETE_SIZE: usize = BLOCK_SIZE; +/// Size of the APFS raw extent located on the target machine. +/// +/// The logical store is always exactly this many bytes. The raw-extent owner +/// opens a larger container and serves only the [`STORE_SIZE`] window at its +/// located base, so a wrong base cannot read past the extent. A wrong window +/// still fails closed: +/// the two root records will not validate and `open_based` returns `ENODATA` +/// rather than serving SEP an unrelated store. A machine with a different extent +/// size must pass that size through an explicit, reviewed interface. +const STORE_SIZE: u64 = 0x600000; +const MAX_SLOTS: usize = 4096; + +/// Sequential read window for the automatic gigalocker search (64 KiB — small +/// enough for a reliable kernel allocation, large enough to keep the scan of +/// the container to a modest number of reads). +const SCAN_CHUNK: usize = 1 << 16; +/// Upper bound on candidate bases confirmed during the automatic search, so a +/// container full of coincidental root-shaped headers cannot spin the scan. The +/// real store's roots sit within the first slots, so it is found long before +/// this; exceeding it falls back rather than looping. +const MAX_LOCATE_ATTEMPTS: u32 = 64; + +pub(crate) const MAX_VALUE: usize = 0x8000; + +const KEY_KIND: usize = 0x01; +const KEY_UUID: usize = 0x02; +const LENGTH: usize = 0x12; +const CRC: usize = 0x16; +const REVISION: usize = 0x1a; +const PAYLOAD: usize = HEADER_SIZE; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) kind: u8, + pub(crate) uuid: [u8; 16], +} + +impl Key { + pub(crate) const fn new(kind: u8, uuid: [u8; 16]) -> Key { + Key { kind, uuid } + } + + pub(crate) const fn root(kind: u8) -> Key { + Key { + kind, + uuid: [0; 16], + } + } +} + +#[derive(Clone, Copy)] +struct Slot { + used: bool, + key: Key, + len: u32, + crc: u32, + revision: u64, +} + +impl Slot { + const FREE: Slot = Slot { + used: false, + key: Key::root(0), + len: 0, + crc: 0, + revision: 0, + }; + + fn matches(&self, key: &Key) -> bool { + self.used && self.key == *key + } +} + +pub(crate) struct Store { + file: shim::StoreFile, + /// Byte offset of the gigalocker window within the opened container: the + /// located extent's offset (an explicit start-sector override sets it). + base: u64, + slots: KVec, + revision: u64, + writes_enabled: bool, + valid_records: usize, + malformed_records: usize, + duplicate_records: usize, + repaired_records: usize, +} + +fn le32(bytes: &[u8], off: usize) -> u32 { + u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]) +} + +fn le64(bytes: &[u8], off: usize) -> u64 { + u64::from_le_bytes([ + bytes[off], + bytes[off + 1], + bytes[off + 2], + bytes[off + 3], + bytes[off + 4], + bytes[off + 5], + bytes[off + 6], + bytes[off + 7], + ]) +} + +fn valid_key(key: &Key) -> bool { + (1..=4).contains(&key.kind) && (key.kind > 2 || key.uuid == [0; 16]) +} + +impl Store { + /// Opens the store as the in-kernel raw-extent owner: it opens the iBoot + /// system container directly and serves only the gigalocker extent, with no + /// external helper and no hand-supplied sector. + /// + /// `start_sector == 0` (the default) *locates the gigalocker automatically* + /// inside the container by its CRC-checked root records; a non-zero + /// `start_sector` is an explicit override for the rare case the scan should + /// be skipped. + pub(crate) fn open_owner(writes_enabled: bool, start_sector: u64) -> Result { + if start_sector != 0 { + return Self::open_based(OWNER_PATH, writes_enabled, start_sector << 9); + } + Self::open_located(writes_enabled) + } + + /// Locates the gigalocker inside the iBoot system container with no external + /// help. The container is read sequentially and searched, at block-aligned + /// offsets, for a root record's signature — a `1` or `2` key kind with an + /// all-zero UUID, the shape of [`Key::root`]. Each base implied by such a hit + /// is probed read-only with the full CRC-checked [`open_based`]; because a + /// base shifted a few slots off the true origin still keeps both roots in its + /// window (passing the two-root test while dropping records that fall + /// outside), the base is pinned by the candidate that recovers the MOST live + /// records — the exact discriminator, since only the true origin captures the + /// whole set and container noise never forges a CRC-valid record. Repair is + /// never armed during discovery; only the pinned base is re-opened writable. + /// Bounded by [`MAX_LOCATE_ATTEMPTS`] probes; returns `ENODATA` if not found. + fn open_located(writes_enabled: bool) -> Result { + // The container is a writable block device; the block layer only grants + // a read-only handle to a read-only device, so scan it through a writable + // handle. This does not arm any write — discovery never mutates the + // container (see open_based_ext, called below with arm_writes == false). + let probe = shim::StoreFile::open_block(OWNER_PATH, true)?; + let size = probe.size()?; + if size < STORE_SIZE { + return Err(ENODATA); + } + + let mut buf: KVec = KVec::new(); + buf.resize(SCAN_CHUNK, 0, GFP_KERNEL)?; + let mut attempts: u32 = 0; + let mut off: u64 = 0; + + while off + HEADER_SIZE as u64 <= size { + let want = core::cmp::min(SCAN_CHUNK as u64, size - off) as usize; + probe.read_exact(off, &mut buf[..want])?; + + let mut p = 0usize; + while p + HEADER_SIZE <= want { + let kind = buf[p + KEY_KIND]; + let uuid_zero = buf[p + KEY_UUID..p + KEY_UUID + 16].iter().all(|&b| b == 0); + if (kind == 1 || kind == 2) && uuid_zero { + // A root signature anchors the gigalocker: the true base is + // this hit minus a whole number of slots (slots and blocks are + // both 0x1000-aligned, so every candidate stays block-aligned). + // The two-root test alone does NOT pin the base — a base + // shifted off the true origin by a few slots still keeps both + // roots inside its 6 MiB window, so it passes yet silently + // drops the live records that fall outside the shifted window. + // Probe every candidate READ-ONLY (writes_enabled forced false, + // so repair never fires at an unconfirmed base) and pin the one + // that recovers the MOST live records: only the true origin + // captures the whole record set, and container noise never + // forges a CRC-valid record, so max live records is exact. + let hit = off + p as u64; + let mut best: Option<(usize, usize, u64)> = None; // (valid, malformed, base) + let mut k: u64 = 0; + while k as usize <= MAX_SLOTS { + let step = k * SLOT_SIZE as u64; + if step > hit { + break; + } + let base = hit - step; + k += 1; + if base + STORE_SIZE > size { + continue; + } + attempts += 1; + if attempts > MAX_LOCATE_ATTEMPTS { + break; + } + if let Ok(store) = Self::open_based_ext(OWNER_PATH, true, false, base) { + let cand = (store.valid_records, store.malformed_records, base); + let better = match best { + None => true, + // More live records wins; ties break to fewer + // malformed, then to the HIGHER base. An up-shift + // drops the lowest occupied slots (and any root + // there, which fails the two-root test), so it + // never ties on live count; the only bases that + // can tie are down-shifts, which are all lower + // than the true origin — so the highest surviving + // candidate is the true origin. + Some(b) => { + cand.0 > b.0 + || (cand.0 == b.0 && cand.1 < b.1) + || (cand.0 == b.0 && cand.1 == b.1 && cand.2 > b.2) + } + }; + if better { + best = Some(cand); + } + } + } + // The gigalocker is unique, so the first hit that confirms any + // base has pinned it. Re-open the winner through a writable + // handle and arm repair per `writes_enabled` — repair runs + // now, and only at this confirmed origin. + if let Some((_, _, base)) = best { + return Self::open_based_ext(OWNER_PATH, true, writes_enabled, base); + } + if attempts > MAX_LOCATE_ATTEMPTS { + return Err(ENODATA); + } + } + p += BLOCK_SIZE; + } + + if want < SCAN_CHUNK { + break; + } + // Overlap one block so a signature on the boundary is not missed. + off += (SCAN_CHUNK - BLOCK_SIZE) as u64; + } + Err(ENODATA) + } + + /// Opens a caller-selected block mapping at offset zero. + /// + /// The separate xART self-test module uses this entry point with its fixed + /// loop-only mapper name, so it can execute the real parser and write + /// ordering without enabling SEP. + #[allow(dead_code)] + pub(crate) fn open_at(path: &CStr, writes_enabled: bool) -> Result { + Self::open_based(path, writes_enabled, 0) + } + + /// Opens `path` and serves the [`STORE_SIZE`] window starting at byte + /// `base`. The window must be block-aligned and fit within the device; the + /// logical store is always exactly [`STORE_SIZE`], so a larger backing + /// device (the raw container) serves only its gigalocker extent. + /// + /// The block handle's writability equals `writes_enabled`, and repair runs + /// when writes are enabled. For the raw-extent owner, discovery instead + /// needs a writable handle (the block layer only grants a read-only handle + /// to a read-only device, and the container is writable) *without* arming + /// repair at an unconfirmed base — see [`open_based_ext`]. + fn open_based(path: &CStr, writes_enabled: bool, base: u64) -> Result { + Self::open_based_ext(path, writes_enabled, writes_enabled, base) + } + + /// As [`open_based`], but with the block-handle writability (`dev_writable`) + /// decoupled from whether repair writes are armed (`arm_writes`). Discovery + /// of the raw-extent owner opens the writable container with + /// `dev_writable == true` yet `arm_writes == false`, so a candidate base is + /// fully scanned and CRC-validated without a single write landing at an + /// origin that has not yet been confirmed as the true gigalocker. + fn open_based_ext( + path: &CStr, + dev_writable: bool, + arm_writes: bool, + base: u64, + ) -> Result { + let file = shim::StoreFile::open_block(path, dev_writable)?; + let size = file.size()?; + if base % BLOCK_SIZE as u64 != 0 || STORE_SIZE % BLOCK_SIZE as u64 != 0 { + return Err(EINVAL); + } + let end = base.checked_add(STORE_SIZE).ok_or(EINVAL)?; + if end > size { + return Err(EINVAL); + } + let count = (STORE_SIZE / SLOT_SIZE as u64) as usize; + if count == 0 || count > MAX_SLOTS { + return Err(EINVAL); + } + + let mut slots = KVec::with_capacity(count, GFP_KERNEL)?; + for _ in 0..count { + slots.push(Slot::FREE, GFP_KERNEL)?; + } + let mut store = Store { + file, + base, + slots, + revision: 0, + // Discovery is always read-only. Do not arm even repair writes + // until the required root records have validated. + writes_enabled: false, + valid_records: 0, + malformed_records: 0, + duplicate_records: 0, + repaired_records: 0, + }; + store.scan()?; + // Serving an empty or unrelated 6 MiB mapping is the failure mode that + // originally desynchronised SEP's anti-replay state. This Linux driver is never + // the authority that provisions a blank device-wide store. Require + // both existing root families before any mailbox registration can run. + if store.find(&Key::root(1)).is_none() || store.find(&Key::root(2)).is_none() { + return Err(ENODATA); + } + store.writes_enabled = arm_writes; + if arm_writes { + store.repair_disk()?; + } + Ok(store) + } + + fn slot_offset(slot: usize) -> u64 { + (slot * SLOT_SIZE) as u64 + } + + fn scan(&mut self) -> Result<()> { + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + + for idx in 0..self.slots.len() { + self.file + .read_block_exact(self.base + Self::slot_offset(idx), &mut raw)?; + let kind = raw[KEY_KIND]; + if kind == 0 { + continue; + } + + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); + let key = Key { kind, uuid }; + let len = le32(&raw, LENGTH) as usize; + let crc = le32(&raw, CRC); + let revision = le64(&raw, REVISION); + let valid = valid_key(&key) + && (1..=MAX_VALUE).contains(&len) + && crc32_ieee(&raw[PAYLOAD..PAYLOAD + len.min(MAX_VALUE)]) == crc; + + if !valid { + self.malformed_records += 1; + continue; + } + + self.revision = self.revision.max(revision); + let candidate = Slot { + used: true, + key, + len: len as u32, + crc, + revision, + }; + + if let Some(old) = self.find(&key) { + self.duplicate_records += 1; + // Equal revisions keep the later physical slot: the forward + // scan resolves a tie to the last writer. + if revision >= self.slots[old].revision { + self.slots[old] = Slot::FREE; + self.slots[idx] = candidate; + } + } else { + self.slots[idx] = candidate; + } + } + + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(()) + } + + /// Removes malformed records and duplicate losers only after the mapping + /// has passed its complete read-only scan and both root records exist. + fn repair_disk(&mut self) -> Result<()> { + let mut header = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; + header.resize(DELETE_SIZE, 0, GFP_KERNEL)?; + + for idx in 0..self.slots.len() { + if self.slots[idx].used { + continue; + } + self.file + .read_block_exact(self.base + Self::slot_offset(idx), &mut header)?; + if header[KEY_KIND] != 0 { + self.delete_slot(idx)?; + self.repaired_records += 1; + } + } + Ok(()) + } + + fn find(&self, key: &Key) -> Option { + let mut best: Option = None; + for (idx, slot) in self.slots.iter().enumerate() { + if !slot.matches(key) { + continue; + } + if best.is_none_or(|old| slot.revision >= self.slots[old].revision) { + best = Some(idx); + } + } + best + } + + fn find_free(&self, skip: usize) -> Option { + self.slots + .iter() + .enumerate() + .filter(|(_, slot)| !slot.used) + .nth(skip) + .map(|(idx, _)| idx) + } + + fn delete_slot(&self, slot: usize) -> Result<()> { + let mut zero = KVec::with_capacity(DELETE_SIZE, GFP_KERNEL)?; + zero.resize(DELETE_SIZE, 0, GFP_KERNEL)?; + self.file + .write_block_exact(self.base + Self::slot_offset(slot), &zero)?; + self.file.sync() + } + + pub(crate) fn read(&mut self, key: &Key) -> Result>> { + if !valid_key(key) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(None); + }; + let slot = self.slots[idx]; + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + self.file + .read_block_exact(self.base + Self::slot_offset(idx), &mut raw)?; + + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&raw[KEY_UUID..KEY_UUID + 16]); + let disk_key = Key { + kind: raw[KEY_KIND], + uuid, + }; + let len = le32(&raw, LENGTH) as usize; + if disk_key != *key + || len != slot.len as usize + || le32(&raw, CRC) != slot.crc + || le64(&raw, REVISION) != slot.revision + || crc32_ieee(&raw[PAYLOAD..PAYLOAD + len]) != slot.crc + { + return Err(EIO); + } + + let mut value = KVec::with_capacity(len, GFP_KERNEL)?; + value.extend_from_slice(&raw[PAYLOAD..PAYLOAD + len], GFP_KERNEL)?; + Ok(Some(value)) + } + + pub(crate) fn write(&mut self, key: &Key, value: &[u8]) -> Result<()> { + if !self.writes_enabled { + return Err(EROFS); + } + if !valid_key(key) || value.is_empty() || value.len() > MAX_VALUE { + return Err(EINVAL); + } + + let old = self.find(key); + // A new key skips the first free slot; a replacement reuses it. + let fresh = self.find_free(usize::from(old.is_none())).ok_or(ENOSPC)?; + let revision = self.revision.checked_add(1).ok_or(EINVAL)?; + let crc = crc32_ieee(value); + + let mut raw = KVec::with_capacity(SLOT_SIZE, GFP_KERNEL)?; + raw.resize(SLOT_SIZE, 0, GFP_KERNEL)?; + raw[KEY_KIND] = key.kind; + raw[KEY_UUID..KEY_UUID + 16].copy_from_slice(&key.uuid); + raw[LENGTH..LENGTH + 4].copy_from_slice(&(value.len() as u32).to_le_bytes()); + raw[CRC..CRC + 4].copy_from_slice(&crc.to_le_bytes()); + raw[REVISION..REVISION + 8].copy_from_slice(&revision.to_le_bytes()); + raw[PAYLOAD..PAYLOAD + value.len()].copy_from_slice(value); + + // The new record becomes authoritative only after its complete slot is + // durable. The old record is then removed and flushed separately. + self.file + .write_block_exact(self.base + Self::slot_offset(fresh), &raw)?; + self.file.sync()?; + self.slots[fresh] = Slot { + used: true, + key: *key, + len: value.len() as u32, + crc, + revision, + }; + self.revision = revision; + + if let Some(old) = old { + let _ = self.delete_slot(old); + self.slots[old] = Slot::FREE; + } + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(()) + } + + pub(crate) fn delete(&mut self, key: &Key) -> Result { + if !valid_key(key) { + return Err(EINVAL); + } + let Some(idx) = self.find(key) else { + return Ok(false); + }; + if !self.writes_enabled { + return Err(EROFS); + } + self.delete_slot(idx)?; + self.slots[idx] = Slot::FREE; + self.valid_records = self.slots.iter().filter(|slot| slot.used).count(); + Ok(true) + } + + /// Byte offset of the served gigalocker window within the opened container: + /// the located extent's offset (an explicit start-sector override sets it). + pub(crate) fn base(&self) -> u64 { + self.base + } + + pub(crate) fn summary(&self) -> (usize, usize, u64, usize, usize, usize, bool) { + ( + self.slots.len(), + self.valid_records, + self.revision, + self.malformed_records, + self.duplicate_records, + self.repaired_records, + self.writes_enabled, + ) + } +} + +/// Reflected CRC-32/ISO-HDLC (the IEEE CRC-32 used in gigalocker records). +pub(crate) const fn crc32_ieee(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + let mut i = 0; + while i < data.len() { + crc ^= data[i] as u32; + let mut bit = 0; + while bit < 8 { + crc = if crc & 1 != 0 { + (crc >> 1) ^ 0xedb8_8320 + } else { + crc >> 1 + }; + bit += 1; + } + i += 1; + } + crc ^ 0xffff_ffff +} + +static_assert!(crc32_ieee(b"123456789") == 0xcbf4_3926); diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 61eefb08d2a7c5..6695e61b081f13 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -99,12 +99,32 @@ #define APPLE_SPI_DELAY_PRE 0x160 #define APPLE_SPI_DELAY_POST 0x168 +#define APPLE_SPI_DELAY_POST_CYCLES 0x194 #define APPLE_SPI_DELAY_ENABLE BIT(0) #define APPLE_SPI_DELAY_NO_INTERBYTE BIT(1) #define APPLE_SPI_DELAY_SET_SCK BIT(4) +#define APPLE_SPI_DELAY_SET_CS BIT(5) #define APPLE_SPI_DELAY_SET_MOSI BIT(6) #define APPLE_SPI_DELAY_SCK_VAL BIT(8) +#define APPLE_SPI_DELAY_CS_VAL BIT(10) +#define APPLE_SPI_DELAY_PRE_ARM BIT(11) #define APPLE_SPI_DELAY_MOSI_VAL BIT(12) +#define APPLE_SPI_DELAY_CYCLES GENMASK(31, 16) + +/* + * These are the values used by AppleSPIMCController's hardware-delay path. + * The pre-delay asserts active-low CS and the post-delay releases it. The + * otherwise undocumented bit 11 is named for its observed role rather than + * assigning it a speculative electrical meaning. + */ +#define APPLE_SPI_DELAY_PRE_FLAGS (APPLE_SPI_DELAY_PRE_ARM | \ + APPLE_SPI_DELAY_SET_CS | \ + APPLE_SPI_DELAY_NO_INTERBYTE | \ + APPLE_SPI_DELAY_ENABLE) +#define APPLE_SPI_DELAY_POST_FLAGS (APPLE_SPI_DELAY_CS_VAL | \ + APPLE_SPI_DELAY_SET_CS | \ + APPLE_SPI_DELAY_NO_INTERBYTE | \ + APPLE_SPI_DELAY_ENABLE) #define APPLE_SPI_FIFO_DEPTH 16 @@ -168,6 +188,7 @@ static void apple_spi_init(struct apple_spi *spi) /* Disable delays */ reg_write(spi, APPLE_SPI_DELAY_PRE, 0); reg_write(spi, APPLE_SPI_DELAY_POST, 0); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); } static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) @@ -193,6 +214,71 @@ static void apple_spi_set_cs(struct spi_device *device, bool is_high) reg_mask(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS, is_high ? APPLE_SPI_PIN_CS : 0); } +static int apple_spi_delay_cycles(struct apple_spi *spi, struct spi_delay *delay, + u32 *cycles) +{ + u64 value; + int ns; + + ns = spi_delay_to_ns(delay, NULL); + if (ns < 0) + return ns; + + value = DIV_ROUND_UP_ULL((u64)clk_get_rate(spi->clk) * ns, + 1000000000ULL); + if (value > FIELD_MAX(APPLE_SPI_DELAY_CYCLES)) + return -ERANGE; + + *cycles = value; + return 0; +} + +static int apple_spi_set_cs_timing(struct spi_device *device) +{ + struct apple_spi *spi = spi_controller_get_devdata(device->controller); + u32 setup_cycles, hold_cycles; + int ret; + + /* This controller path has no hardware representation for inactive time. */ + if (device->cs_inactive.value) + return -EOPNOTSUPP; + + ret = apple_spi_delay_cycles(spi, &device->cs_setup, &setup_cycles); + if (ret) + return ret; + ret = apple_spi_delay_cycles(spi, &device->cs_hold, &hold_cycles); + if (ret) + return ret; + + if (!setup_cycles && !hold_cycles) { + reg_mask(spi, APPLE_SPI_SHIFTCFG, + APPLE_SPI_SHIFTCFG_OVERRIDE_CS, 0); + reg_write(spi, APPLE_SPI_DELAY_PRE, 0); + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, 0); + return 0; + } + + reg_write(spi, APPLE_SPI_DELAY_PRE, + FIELD_PREP(APPLE_SPI_DELAY_CYCLES, setup_cycles) | + APPLE_SPI_DELAY_PRE_FLAGS); + reg_write(spi, APPLE_SPI_DELAY_POST_CYCLES, hold_cycles); + reg_write(spi, APPLE_SPI_DELAY_POST, + FIELD_PREP(APPLE_SPI_DELAY_CYCLES, hold_cycles) | + APPLE_SPI_DELAY_POST_FLAGS); + reg_mask(spi, APPLE_SPI_SHIFTCFG, 0, + APPLE_SPI_SHIFTCFG_OVERRIDE_CS); + + dev_dbg(&device->dev, + "hardware CS setup=%u hold=%u cycles: pre=%#08x post=%#08x post_cycles=%#08x shiftcfg=%#08x\n", + setup_cycles, hold_cycles, + reg_read(spi, APPLE_SPI_DELAY_PRE), + reg_read(spi, APPLE_SPI_DELAY_POST), + reg_read(spi, APPLE_SPI_DELAY_POST_CYCLES), + reg_read(spi, APPLE_SPI_SHIFTCFG)); + return 0; +} + static bool apple_spi_prep_transfer(struct apple_spi *spi, struct spi_transfer *t) { u32 cr, fifo_threshold; @@ -491,6 +577,7 @@ static int apple_spi_probe(struct platform_device *pdev) ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); ctlr->prepare_message = apple_spi_prepare_message; ctlr->set_cs = apple_spi_set_cs; + ctlr->set_cs_timing = apple_spi_set_cs_timing; ctlr->transfer_one = apple_spi_transfer_one; ctlr->use_gpio_descriptors = true; ctlr->auto_runtime_pm = true; diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index 03527162613f72..bd9d21f55f6cb3 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -83,6 +83,9 @@ struct trusted_key_source { extern struct key_type key_type_trusted; +int register_trusted_key_source(struct trusted_key_source *src); +void unregister_trusted_key_source(struct trusted_key_source *src); + #define TRUSTED_DEBUG 0 #if TRUSTED_DEBUG diff --git a/include/linux/apple-sep-fv.h b/include/linux/apple-sep-fv.h new file mode 100644 index 00000000000000..1a2a4e4ec09f7d --- /dev/null +++ b/include/linux/apple-sep-fv.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_APPLE_SEP_FV_H +#define _LINUX_APPLE_SEP_FV_H + +#include + +#define APPLE_SEP_FV_OPAQUE_KEY_SIZE 64 +#define APPLE_SEP_FV_IV_KEY_SIZE 16 +#define APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE 168 + +struct apple_sep_fv_key { + u8 opaque[APPLE_SEP_FV_OPAQUE_KEY_SIZE]; + u8 iv[APPLE_SEP_FV_IV_KEY_SIZE]; +}; + +struct apple_sep_fv_new_file_key { + struct apple_sep_fv_key key; + u8 wrapped_ekwk[APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE]; + u8 wrapped_ek[APPLE_SEP_FV_MAX_WRAPPED_KEY_SIZE]; + size_t wrapped_ekwk_len; + size_t wrapped_ek_len; +}; + +int apple_sep_fv_unwrap_media_key(const u8 *wrapped, size_t wrapped_len, + u32 protection_class, + struct apple_sep_fv_key *key); +int apple_sep_fv_unwrap_volume_key(const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len, + struct apple_sep_fv_key *key); +int apple_sep_fv_load_class_keys_v2(const u8 volume_uuid[16], + const u8 *secret, size_t secret_len, + const u8 *unlock_record, + size_t unlock_record_len, + const u8 *volume_key, + size_t volume_key_len); +int apple_sep_fv_unload_class_keys_v2(const u8 volume_uuid[16], + const u8 *volume_key, + size_t volume_key_len); +int apple_sep_fv_unwrap_file_key(const u8 volume_uuid[16], + u32 protection_class, + const u8 *wrapped_ekwk, + size_t wrapped_ekwk_len, + const u8 *wrapped_ek, + size_t wrapped_ek_len, + struct apple_sep_fv_key *key); +int apple_sep_fv_new_file_key_v2(const u8 volume_uuid[16], + u32 protection_class, + u64 crypto_id, u16 key_revision, + struct apple_sep_fv_new_file_key *key); + +#endif diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index 0b142d941cd2e3..143fc687880abb 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -61,13 +61,134 @@ DEFINE_STATIC_CALL_NULL(trusted_key_get_random, static void (*trusted_key_exit)(void); static unsigned char migratable; +static DEFINE_MUTEX(trusted_key_source_lock); +static bool trusted_key_source_active; +static struct trusted_key_source *trusted_key_source_registered; + +/* Defined below; registration needs it before its definition. */ +static int kernel_get_random(unsigned char *key, size_t key_len); + +/* + * Dispatch targets for when no source is registered. The call sites use + * static_call() directly, so the pointers must always be valid code; a NULL + * target would fault instead of returning an error. + */ +static int trusted_key_absent_seal(struct trusted_key_payload *p, char *datablob) +{ + return -ENODEV; +} + +static int trusted_key_absent_unseal(struct trusted_key_payload *p, char *datablob) +{ + return -ENODEV; +} + +static int trusted_key_absent_get_random(unsigned char *key, size_t key_len) +{ + return -ENODEV; +} + +/** + * register_trusted_key_source - offer a trust source from a module + * @src: the source. Must outlive its registration. + * + * Returns 0 on success, -EBUSY if a source is already active, -ENODEV if + * trusted.source= names a different one, or the source's own init() error. + */ +int register_trusted_key_source(struct trusted_key_source *src) +{ + int (*get_random)(unsigned char *key, size_t key_len); + int ret; + + if (!src || !src->name || !src->ops || !src->ops->init || + !src->ops->seal || !src->ops->unseal) + return -EINVAL; + + /* An explicit trusted.source= names one source; honour it. */ + if (trusted_key_source && strcmp(trusted_key_source, src->name)) + return -ENODEV; + + mutex_lock(&trusted_key_source_lock); + + if (trusted_key_source_active) { + mutex_unlock(&trusted_key_source_lock); + return -EBUSY; + } + + get_random = src->ops->get_random; + if (trusted_rng && strcmp(trusted_rng, "default")) { + if (!strcmp(trusted_rng, "kernel")) { + get_random = kernel_get_random; + } else if (strcmp(trusted_rng, src->name) || !get_random) { + mutex_unlock(&trusted_key_source_lock); + return -EINVAL; + } + } + if (!get_random) + get_random = kernel_get_random; + + ret = src->ops->init(); + if (ret) { + mutex_unlock(&trusted_key_source_lock); + return ret; + } + + static_call_update(trusted_key_seal, src->ops->seal); + static_call_update(trusted_key_unseal, src->ops->unseal); + static_call_update(trusted_key_get_random, get_random); + trusted_key_exit = src->ops->exit; + migratable = src->ops->migratable; + trusted_key_source_active = true; + trusted_key_source_registered = src; + + mutex_unlock(&trusted_key_source_lock); + pr_info("trusted_key: source '%s' registered\n", src->name); + return 0; +} +EXPORT_SYMBOL_GPL(register_trusted_key_source); + +/** + * unregister_trusted_key_source - withdraw a previously registered source + * @src: the source that was registered. + * + * Keys sealed by @src stay in the keyring and stop being usable, which is the + * intended outcome: the thing that could unseal them is gone. + */ +void unregister_trusted_key_source(struct trusted_key_source *src) +{ + mutex_lock(&trusted_key_source_lock); + + /* Only the source that registered may withdraw itself. */ + if (!trusted_key_source_active || src != trusted_key_source_registered) { + mutex_unlock(&trusted_key_source_lock); + return; + } + + static_call_update(trusted_key_seal, trusted_key_absent_seal); + static_call_update(trusted_key_unseal, trusted_key_absent_unseal); + static_call_update(trusted_key_get_random, trusted_key_absent_get_random); + + if (trusted_key_exit) + trusted_key_exit(); + trusted_key_exit = NULL; + migratable = 0; + trusted_key_source_active = false; + trusted_key_source_registered = NULL; + + mutex_unlock(&trusted_key_source_lock); + pr_info("trusted_key: source '%s' unregistered\n", + src && src->name ? src->name : "?"); +} +EXPORT_SYMBOL_GPL(unregister_trusted_key_source); + enum { Opt_err, - Opt_new, Opt_load, Opt_update, + Opt_new, Opt_import, Opt_load, Opt_update, }; static const match_table_t key_tokens = { {Opt_new, "new"}, + {Opt_import, "import"}, {Opt_load, "load"}, {Opt_update, "update"}, {Opt_err, NULL} @@ -104,6 +225,19 @@ static int datablob_parse(char **datablob, struct trusted_key_payload *p) p->key_len = keylen; ret = Opt_new; break; + case Opt_import: + c = strsep(datablob, " \t"); + if (!c || strlen(c) % 2) + return -EINVAL; + keylen = strlen(c) / 2; + if (keylen < MIN_KEY_SIZE || keylen > MAX_KEY_SIZE) + return -EINVAL; + ret = hex2bin(p->key, c, keylen); + if (ret < 0) + return -EINVAL; + p->key_len = keylen; + ret = Opt_import; + break; case Opt_load: /* first argument is sealed blob */ c = strsep(datablob, " \t"); @@ -205,6 +339,11 @@ static int trusted_instantiate(struct key *key, goto out; } + ret = static_call(trusted_key_seal)(payload, datablob); + if (ret < 0) + pr_info("key_seal failed (%d)\n", ret); + break; + case Opt_import: ret = static_call(trusted_key_seal)(payload, datablob); if (ret < 0) pr_info("key_seal failed (%d)\n", ret); @@ -337,7 +476,10 @@ static int kernel_get_random(unsigned char *key, size_t key_len) static int __init init_trusted(void) { int (*get_random)(unsigned char *key, size_t key_len); - int i, ret = 0; + /* -ENODEV when the built-in array is empty or nothing matches, so a + * later module source can still register instead of hitting -EBUSY. + */ + int i, ret = -ENODEV; for (i = 0; i < ARRAY_SIZE(trusted_key_sources); i++) { if (trusted_key_source && @@ -385,8 +527,21 @@ static int __init init_trusted(void) * encrypted_keys.ko depends on successful load of this module even if * trusted key implementation is not found. */ - if (ret == -ENODEV) + if (ret == -ENODEV) { + /* + * No built-in source. Leave the dispatch pointing at code that + * fails rather than at NULL, so a seal attempted before any + * module registers returns an error instead of faulting. + */ + static_call_update(trusted_key_seal, trusted_key_absent_seal); + static_call_update(trusted_key_unseal, trusted_key_absent_unseal); + static_call_update(trusted_key_get_random, + trusted_key_absent_get_random); return 0; + } + + if (!ret) + trusted_key_source_active = true; return ret; } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md new file mode 100644 index 00000000000000..6c28a65403f51f --- /dev/null +++ b/tools/aurora-sep/README.md @@ -0,0 +1,34 @@ +# Aurora SEP userspace integration + +These files connect the Apple SEP kernel driver to the shared APFS xART +gigalocker and the desktop fingerprint stack. + +Install the driver service: + +```sh +install -Dm755 load-driver /usr/local/sbin/aurora-sep-load +install -Dm644 aurora-sep.service /etc/systemd/system/aurora-sep.service +systemctl daemon-reload +systemctl enable aurora-sep.service +``` + +The kernel driver locates the xART gigalocker itself inside the iBoot System +Container (`modprobe apple_sep xart_writes=1 provision_keybag=1`, with no +`xart_start_sector`): it opens the container directly and serves only the +gigalocker extent, needing no external helper and no hand-supplied sector. + +For fingerprint support, apply +`patches/libfprint-1.94.100-apple-sep.patch` to libfprint 1.94.100, build and +install libfprint, then install the fprintd device policy: + +```sh +install -Dm644 fprintd-aurora.conf \ + /etc/systemd/system/fprintd.service.d/aurora.conf +systemctl daemon-reload +systemctl restart fprintd.service +``` + +On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell +lock screen then selects `omarchy-lock-fingerprint` automatically and accepts +Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used +by the Quickshell lock screen. diff --git a/tools/aurora-sep/aurora-sep.service b/tools/aurora-sep/aurora-sep.service new file mode 100644 index 00000000000000..4287247cba0da1 --- /dev/null +++ b/tools/aurora-sep/aurora-sep.service @@ -0,0 +1,13 @@ +[Unit] +Description=Aurora SEP services +After=local-fs.target +RequiresMountsFor=/var/lib + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecCondition=/bin/sh -c 'modinfo apple_sep >/dev/null 2>&1' +ExecStart=/usr/local/sbin/aurora-sep-load + +[Install] +WantedBy=multi-user.target diff --git a/tools/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/fprintd-aurora.conf new file mode 100644 index 00000000000000..6b33af25709488 --- /dev/null +++ b/tools/aurora-sep/fprintd-aurora.conf @@ -0,0 +1,6 @@ +[Unit] +After=aurora-sep.service +Requires=aurora-sep.service + +[Service] +DeviceAllow=/dev/sep-bio rw diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver new file mode 100755 index 00000000000000..f5579fc01b360a --- /dev/null +++ b/tools/aurora-sep/load-driver @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu + +keybag=/var/lib/aurora-sep-keybag.bin +refkey=/var/lib/aurora-sep-refkey.bin + +modprobe apple_sep xart_writes=1 provision_keybag=1 + +work=$(mktemp -d /run/aurora-sep-load.XXXXXX) +cleanup() { + case "$work" in + /run/aurora-sep-load.*) rm -rf -- "$work" ;; + esac +} +trap cleanup EXIT HUP INT TERM + +keyctl new_session aurora-sep-load >/dev/null +attempts=300 +verified=false +while [ "$attempts" -gt 0 ]; do + key_id=$(printf 'new 32' | keyctl padd trusted aurora-sep-check @s 2>/dev/null || true) + if [ -n "$key_id" ]; then + if keyctl pipe "$key_id" >"$work/sealed"; then + { printf 'load '; tr -d '\n\r ' <"$work/sealed"; } >"$work/load" + loaded_id=$(keyctl padd trusted aurora-sep-load-check @s <"$work/load" 2>/dev/null || true) + if [ -n "$loaded_id" ]; then + keyctl revoke "$loaded_id" >/dev/null 2>&1 || true + verified=true + fi + fi + keyctl revoke "$key_id" >/dev/null 2>&1 || true + fi + [ "$verified" = true ] && break + sleep 0.1 + attempts=$((attempts - 1)) +done + +[ "$verified" = true ] || exit 1 +[ -s "$keybag" ] || exit 1 +[ -s "$refkey" ] || exit 1 + +attempts=300 +while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do + sleep 0.1 + attempts=$((attempts - 1)) +done +[ -c /dev/sep-bio ] || exit 1 diff --git a/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch new file mode 100644 index 00000000000000..37593f65ea05e0 --- /dev/null +++ b/tools/aurora-sep/patches/libfprint-1.94.100-apple-sep.patch @@ -0,0 +1,1179 @@ +diff --git a/libfprint/drivers/aurora/aurora-bio.h b/libfprint/drivers/aurora/aurora-bio.h +new file mode 100644 +index 0000000..b2d2f43 +--- /dev/null ++++ b/libfprint/drivers/aurora/aurora-bio.h +@@ -0,0 +1,129 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Copyright 2026 Dj */ ++/* ++ * Userspace interface for the SEP/Mesa biometric device. ++ * ++ * The enclave does the matching. Nothing biometric crosses this interface: only ++ * an operation, a stage, a status, an opaque identity UUID, and an opaque host ++ * label userspace chooses. ++ * ++ * SPDX-License-Identifier: LGPL-2.1-or-later ++ */ ++ ++#pragma once ++ ++#include ++#include ++ ++#define AURORA_BIO_IFACE_VERSION 4 ++ ++#define AURORA_BIO_UUID_LEN 16 ++#define AURORA_BIO_LABEL_LEN 128 ++#define AURORA_BIO_NONCE_LEN 32 ++#define AURORA_BIO_TOKEN_LEN 32 ++#define AURORA_BIO_MAX_IDENTITIES 32 ++#define AURORA_BIO_CHALLENGE_LEN 32 ++#define AURORA_BIO_ATTEST_PUB_LEN 65 ++#define AURORA_BIO_ATTEST_SIG_MAX 72 ++ ++enum { ++ AURORA_BIO_STATE_IDLE = 0, ++ AURORA_BIO_STATE_PENDING = 1, ++ AURORA_BIO_STATE_PROGRESS = 2, ++ AURORA_BIO_STATE_DONE = 3, ++ AURORA_BIO_STATE_FAILED = 4, ++}; ++ ++ ++enum { ++ AURORA_BIO_NO_MATCH = 0, ++ AURORA_BIO_MATCH = 1, ++ AURORA_BIO_NOT_COMPARED = 2, ++}; ++ ++struct aurora_bio_identity { ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u8 label[AURORA_BIO_LABEL_LEN]; ++}; ++ ++struct aurora_bio_info { ++ __u32 version; ++ __u32 sensor_present; ++ __u32 enrolled; ++ __u32 capacity; ++ __u32 enroll_stages; ++ __u32 reserved[3]; ++}; ++ ++struct aurora_bio_list { ++ __u32 count; ++ __u32 reserved; ++ struct aurora_bio_identity id[AURORA_BIO_MAX_IDENTITIES]; ++}; ++ ++struct aurora_bio_enrol_start { ++ __u32 flags; ++ __u32 reserved; ++ __u8 label[AURORA_BIO_LABEL_LEN]; ++}; ++ ++#define AURORA_BIO_GUIDANCE_NONE 0 ++#define AURORA_BIO_GUIDANCE_PLACE 1 ++#define AURORA_BIO_GUIDANCE_LIFT_AND_MOVE 2 ++#define AURORA_BIO_GUIDANCE_HOLD_STILL 3 ++ ++struct aurora_bio_enrol_poll { ++ __u32 state; ++ __u32 stage; ++ __u32 stages_total; ++ __u32 status; ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u32 guidance; ++ __u32 progress_percent; ++}; ++struct aurora_bio_verify_start { ++ __u32 flags; ++ __u32 reserved; ++ __u8 nonce[AURORA_BIO_NONCE_LEN]; ++}; ++ ++struct aurora_bio_verify_poll { ++ __u32 state; ++ __u32 result; ++ __u32 status; ++ __u32 reserved; ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++ __u8 token[AURORA_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ ++ __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ ++}; ++ ++struct aurora_bio_delete { ++ __u8 uuid[AURORA_BIO_UUID_LEN]; ++}; ++ ++/* ++ * Device attestation of key possession: the enclave signs 'challenge' with the ++ * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and ++ * returns the DER signature and public point. The private key never leaves the ++ * enclave. ++ */ ++struct aurora_bio_attest { ++ __u32 sig_len; /* out: DER signature length */ ++ __u8 challenge[AURORA_BIO_CHALLENGE_LEN]; /* in */ ++ __u8 public[AURORA_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ ++ __u8 signature[AURORA_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ ++ __u8 reserved[3]; ++}; ++ ++#define AURORA_BIO_IOC_MAGIC 0xB1 ++ ++#define AURORA_BIO_GET_INFO _IOR (AURORA_BIO_IOC_MAGIC, 0x01, struct aurora_bio_info) ++#define AURORA_BIO_LIST _IOR (AURORA_BIO_IOC_MAGIC, 0x02, struct aurora_bio_list) ++#define AURORA_BIO_ENROL_START _IOW (AURORA_BIO_IOC_MAGIC, 0x03, struct aurora_bio_enrol_start) ++#define AURORA_BIO_ENROL_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x04, struct aurora_bio_enrol_poll) ++#define AURORA_BIO_VERIFY_START _IOW (AURORA_BIO_IOC_MAGIC, 0x05, struct aurora_bio_verify_start) ++#define AURORA_BIO_VERIFY_POLL _IOR (AURORA_BIO_IOC_MAGIC, 0x06, struct aurora_bio_verify_poll) ++#define AURORA_BIO_CANCEL _IO (AURORA_BIO_IOC_MAGIC, 0x07) ++#define AURORA_BIO_DELETE _IOW (AURORA_BIO_IOC_MAGIC, 0x08, struct aurora_bio_delete) ++#define AURORA_BIO_DELETE_ALL _IO (AURORA_BIO_IOC_MAGIC, 0x09) ++#define AURORA_BIO_ATTEST _IOWR(AURORA_BIO_IOC_MAGIC, 0x0a, struct aurora_bio_attest) +diff --git a/libfprint/drivers/aurora/aurora.c b/libfprint/drivers/aurora/aurora.c +new file mode 100644 +index 0000000..55f3311 +--- /dev/null ++++ b/libfprint/drivers/aurora/aurora.c +@@ -0,0 +1,829 @@ ++/* ++ * Aurora SEP fingerprint driver ++ * ++ * The sensor is matched inside Apple's secure enclave. This driver never sees ++ * an image, a template, or key material: it starts operations on the kernel ++ * device and reports the enclave's verdict. That is why it can be short. ++ * ++ * SPDX-License-Identifier: LGPL-2.1-or-later ++ */ ++ ++#define FP_COMPONENT "apple-sep" ++ ++#include "drivers_api.h" ++#include "aurora-bio.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++struct _FpiDeviceAurora ++{ ++ FpDevice parent; ++ ++ gint fd; ++ guint watch_id; ++ ++ guint8 nonce[AURORA_BIO_NONCE_LEN]; ++ gboolean nonce_valid; ++}; ++ ++G_DECLARE_FINAL_TYPE (FpiDeviceAurora, fpi_device_aurora, FPI, DEVICE_AURORA, FpDevice) ++G_DEFINE_TYPE (FpiDeviceAurora, fpi_device_aurora, FP_TYPE_DEVICE) ++ ++/* ------------------------------------------------------------------ */ ++/* helpers */ ++/* ------------------------------------------------------------------ */ ++ ++static gboolean ++aurora_ioctl (FpiDeviceAurora *self, unsigned long req, void *arg, GError **error) ++{ ++ if (self->fd < 0) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_OPEN, ++ "device is not open"); ++ return FALSE; ++ } ++ ++ if (ioctl (self->fd, req, arg) < 0) ++ { ++ gint err = errno; ++ ++ switch (err) ++ { ++ case ENODEV: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_REMOVED, ++ "the enclave reports no fingerprint sensor"); ++ break; ++ ++ case EPERM: ++ case EACCES: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "not permitted"); ++ break; ++ ++ case ENOSPC: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_FULL, ++ "no space for another identity"); ++ break; ++ ++ case ENOENT: ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_NOT_FOUND, ++ "no such identity"); ++ break; ++ ++ default: ++ g_set_error (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_PROTO, ++ "ioctl failed: %s", g_strerror (err)); ++ break; ++ } ++ return FALSE; ++ } ++ ++ return TRUE; ++} ++ ++/* The label is how the host recognises its own records in the device's list. ++ * libfprint already has an encoding for exactly this -- it carries the enroll ++ * date, the finger and the username -- so use it rather than inventing one. */ ++static gboolean ++aurora_label_from_print (FpPrint *print, guint8 *label_out, GError **error) ++{ ++ g_autofree gchar *user_id = NULL; ++ ++ if (print == NULL) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "no print to label"); ++ return FALSE; ++ } ++ ++ user_id = fpi_print_generate_user_id (print); ++ ++ if (user_id == NULL || strlen (user_id) >= AURORA_BIO_LABEL_LEN) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "print label does not fit the device record"); ++ return FALSE; ++ } ++ ++ memset (label_out, 0, AURORA_BIO_LABEL_LEN); ++ memcpy (label_out, user_id, strlen (user_id)); ++ return TRUE; ++} ++ ++static void ++aurora_stop_watch (FpiDeviceAurora *self) ++{ ++ if (self->watch_id != 0) ++ { ++ g_source_remove (self->watch_id); ++ self->watch_id = 0; ++ } ++} ++ ++/* Build the FpPrint that represents one enclave-stored identity. The identity ++ * UUID is all we hold; there is no template on this side to store. */ ++static FpPrint * ++aurora_print_from_uuid (FpiDeviceAurora *self, ++ const guint8 *uuid, ++ const guint8 *label) ++{ ++ FpPrint *print = fp_print_new (FP_DEVICE (self)); ++ GVariant *data; ++ ++ data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, uuid, ++ AURORA_BIO_UUID_LEN, 1); ++ ++ fpi_print_set_type (print, FPI_PRINT_RAW); ++ fpi_print_set_device_stored (print, TRUE); ++ g_object_set (print, "fpi-data", data, NULL); ++ ++ /* The label came from the device. Treat it as untrusted bytes: bound it to ++ * the buffer before anything reads it as a string. */ ++ if (label != NULL) ++ { ++ gchar safe[AURORA_BIO_LABEL_LEN]; ++ ++ memcpy (safe, label, AURORA_BIO_LABEL_LEN); ++ safe[AURORA_BIO_LABEL_LEN - 1] = '\0'; ++ ++ if (safe[0] != '\0') ++ fpi_print_fill_from_user_id (print, safe); ++ } ++ ++ return print; ++} ++ ++static gboolean ++aurora_uuid_from_print (FpPrint *print, guint8 *uuid_out, GError **error) ++{ ++ g_autoptr(GVariant) data = NULL; ++ const guint8 *raw; ++ gsize len = 0; ++ ++ g_object_get (print, "fpi-data", &data, NULL); ++ ++ if (data == NULL || !g_variant_is_of_type (data, G_VARIANT_TYPE ("ay"))) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "print does not carry an enclave identity"); ++ return FALSE; ++ } ++ ++ raw = g_variant_get_fixed_array (data, &len, 1); ++ if (raw == NULL || len != AURORA_BIO_UUID_LEN) ++ { ++ g_set_error_literal (error, FP_DEVICE_ERROR, FP_DEVICE_ERROR_DATA_INVALID, ++ "enclave identity is the wrong size"); ++ return FALSE; ++ } ++ ++ memcpy (uuid_out, raw, AURORA_BIO_UUID_LEN); ++ return TRUE; ++} ++ ++/* Compare by enclave identity rather than by whole print. ++ * ++ * fp_print_equal() compares host-side labels too -- the username and finger a ++ * print was filed under. Those are ours, not the enclave's: the device knows ++ * only the identity it stored. Comparing whole prints made a correct match ++ * from fprintd, which files prints under a username, read as a non-match ++ * against the bare print the device hands back. The UUID is the ground truth, ++ * so compare that. */ ++static gboolean ++aurora_print_has_uuid (FpPrint *print, const guint8 *uuid) ++{ ++ guint8 stored[AURORA_BIO_UUID_LEN]; ++ ++ if (print == NULL) ++ return FALSE; ++ ++ if (!aurora_uuid_from_print (print, stored, NULL)) ++ return FALSE; ++ ++ return memcmp (stored, uuid, AURORA_BIO_UUID_LEN) == 0; ++} ++ ++/* ------------------------------------------------------------------ */ ++/* probe / open / close */ ++/* ------------------------------------------------------------------ */ ++ ++static void ++aurora_probe (FpDevice *device) ++{ ++ struct aurora_bio_info info = { 0 }; ++ const gchar *path; ++ g_autofree gchar *serial = NULL; ++ gint fd; ++ ++ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); ++ if (path == NULL) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "no sep-bio node")); ++ return; ++ } ++ ++ fd = open (path, O_RDWR | O_CLOEXEC); ++ if (fd < 0) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot open %s: %s", ++ path, g_strerror (errno))); ++ return; ++ } ++ ++ if (ioctl (fd, AURORA_BIO_GET_INFO, &info) < 0) ++ { ++ gint err = errno; ++ ++ close (fd); ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot query device: %s", ++ g_strerror (err))); ++ return; ++ } ++ ++ close (fd); ++ ++ if (info.version != AURORA_BIO_IFACE_VERSION) ++ { ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "interface version %u, expected %u", ++ info.version, ++ AURORA_BIO_IFACE_VERSION)); ++ return; ++ } ++ ++ /* A driver that claims a device it cannot use makes the whole stack look ++ * broken to the user. If the enclave has no sensor, say so plainly and let ++ * fprintd report "no devices" rather than a device that fails every touch. */ ++ if (!info.sensor_present) ++ { ++ fp_dbg ("enclave reports no fingerprint sensor attached"); ++ fpi_device_probe_complete (device, NULL, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "the secure enclave reports no " ++ "fingerprint sensor attached")); ++ return; ++ } ++ ++ if (info.enroll_stages > 0) ++ fpi_device_set_nr_enroll_stages (device, info.enroll_stages); ++ ++ serial = g_strdup ("apple-sep"); ++ fpi_device_probe_complete (device, serial, NULL, NULL); ++} ++ ++static void ++aurora_open (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ const gchar *path; ++ ++ path = fpi_device_get_udev_data (device, FPI_DEVICE_UDEV_SUBTYPE_MISC); ++ if (path == NULL) ++ { ++ fpi_device_open_complete (device, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "no sep-bio node")); ++ return; ++ } ++ ++ /* The kernel admits one opener at a time; EBUSY means something else holds ++ * the sensor, which is a real condition and not a driver fault. */ ++ self->fd = open (path, O_RDWR | O_CLOEXEC); ++ if (self->fd < 0) ++ { ++ gint err = errno; ++ ++ fpi_device_open_complete (device, ++ fpi_device_error_new_msg (err == EBUSY ++ ? FP_DEVICE_ERROR_BUSY ++ : FP_DEVICE_ERROR_NOT_SUPPORTED, ++ "cannot open %s: %s", ++ path, g_strerror (err))); ++ return; ++ } ++ ++ fpi_device_open_complete (device, NULL); ++} ++ ++static void ++aurora_close (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ ++ aurora_stop_watch (self); ++ ++ if (self->fd >= 0) ++ { ++ /* Closing the description cancels anything in flight, by contract. */ ++ close (self->fd); ++ self->fd = -1; ++ } ++ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ fpi_device_close_complete (device, NULL); ++} ++ ++static void ++aurora_cancel (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ ++ if (self->fd >= 0) ++ ioctl (self->fd, AURORA_BIO_CANCEL); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* enrolment */ ++/* ------------------------------------------------------------------ */ ++ ++static gboolean ++aurora_enrol_ready (gint fd, GIOCondition condition, gpointer user_data) ++{ ++ FpDevice *device = FP_DEVICE (user_data); ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_enrol_poll poll_res = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_ENROL_POLL, &poll_res, &error)) ++ { ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return G_SOURCE_REMOVE; ++ } ++ ++ switch (poll_res.state) ++ { ++ case AURORA_BIO_STATE_PENDING: ++ return G_SOURCE_CONTINUE; ++ ++ case AURORA_BIO_STATE_PROGRESS: ++ fpi_device_enroll_progress (device, poll_res.stage, NULL, NULL); ++ return G_SOURCE_CONTINUE; ++ ++ case AURORA_BIO_STATE_DONE: ++ fpi_device_get_enroll_data (device, &print); ++ if (print != NULL) ++ { ++ GVariant *data = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, ++ poll_res.uuid, ++ AURORA_BIO_UUID_LEN, 1); ++ fpi_print_set_type (print, FPI_PRINT_RAW); ++ fpi_print_set_device_stored (print, TRUE); ++ g_object_set (print, "fpi-data", data, NULL); ++ } ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, ++ print ? g_object_ref (print) : NULL, ++ print ? NULL ++ : fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "enrolment produced no print")); ++ return G_SOURCE_REMOVE; ++ ++ case AURORA_BIO_STATE_FAILED: ++ default: ++ self->watch_id = 0; ++ fpi_device_enroll_complete (device, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "enrolment failed, enclave status 0x%x", ++ poll_res.status)); ++ return G_SOURCE_REMOVE; ++ } ++} ++ ++static void ++aurora_enroll (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_enrol_start start = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ fpi_device_get_enroll_data (device, &print); ++ ++ if (!aurora_label_from_print (print, start.label, &error)) ++ { ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (!aurora_ioctl (self, AURORA_BIO_ENROL_START, &start, &error)) ++ { ++ fpi_device_enroll_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ aurora_stop_watch (self); ++ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_enrol_ready, device); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* verify and identify */ ++/* ------------------------------------------------------------------ */ ++ ++/* The single place a match may be declared. ++ * ++ * Everything else in this driver reports failure. A result counts as a match ++ * only if the enclave said DONE and said MATCH, we require it to have bound the ++ * answer to the nonce we generated for this operation (a binding the host ++ * cannot verify cryptographically -- it is enforced by the enclave), and it ++ * arrived within the token's deadline. Any other combination -- including a ++ * state we do not recognise -- is a non-match, never an error that a caller ++ * might interpret loosely. */ ++static gboolean ++aurora_result_is_match (FpiDeviceAurora *self, ++ const struct aurora_bio_verify_poll *res) ++{ ++ struct timespec now; ++ guint64 now_ns; ++ gboolean token_set = FALSE; ++ gsize i; ++ ++ if (res->state != AURORA_BIO_STATE_DONE) ++ return FALSE; ++ ++ if (res->result != AURORA_BIO_MATCH) ++ return FALSE; ++ ++ if (!self->nonce_valid) ++ return FALSE; ++ ++ for (i = 0; i < AURORA_BIO_TOKEN_LEN; i++) ++ if (res->token[i] != 0) ++ { ++ token_set = TRUE; ++ break; ++ } ++ ++ if (!token_set) ++ return FALSE; ++ ++ if (clock_gettime (CLOCK_MONOTONIC, &now) != 0) ++ return FALSE; ++ ++ now_ns = (guint64) now.tv_sec * 1000000000ull + (guint64) now.tv_nsec; ++ if (res->deadline_ns == 0 || now_ns > res->deadline_ns) ++ { ++ fp_dbg ("match result arrived after its deadline; refusing it"); ++ return FALSE; ++ } ++ ++ return TRUE; ++} ++ ++static gboolean ++aurora_verify_ready (gint fd, GIOCondition condition, gpointer user_data) ++{ ++ FpDevice *device = FP_DEVICE (user_data); ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_verify_poll res = { 0 }; ++ FpiDeviceAction action = fpi_device_get_current_action (device); ++ g_autoptr(GError) error = NULL; ++ /* Not a g_autoptr: FpPrint is a GInitiallyUnowned and both report functions ++ * take the floating reference. Unreffing it here as well drops the last ++ * reference under libfprint's feet, and it frees the print again when the ++ * match completes. */ ++ FpPrint *matched = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_POLL, &res, &error)) ++ { ++ self->watch_id = 0; ++ /* Terminal path: void the per-operation nonce like every other exit, so ++ * a nonce cannot be carried into a later operation. */ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return G_SOURCE_REMOVE; ++ } ++ ++ if (res.state == AURORA_BIO_STATE_PENDING || ++ res.state == AURORA_BIO_STATE_PROGRESS) ++ return G_SOURCE_CONTINUE; ++ ++ self->watch_id = 0; ++ ++ if (aurora_result_is_match (self, &res)) ++ matched = aurora_print_from_uuid (self, res.uuid, NULL); ++ ++ ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ { ++ FpPrint *expected = NULL; ++ ++ fpi_device_get_verify_data (device, &expected); ++ ++ if (matched != NULL && aurora_print_has_uuid (expected, res.uuid)) ++ { ++ fpi_device_verify_report (device, FPI_MATCH_SUCCESS, matched, NULL); ++ fpi_device_verify_complete (device, NULL); ++ } ++ else if (res.result == AURORA_BIO_NOT_COMPARED) ++ { ++ /* No comparison happened, so there is no verdict to report. Saying ++ * FAIL here would tell the user their finger was rejected when it ++ * was never examined. */ ++ fpi_device_verify_complete (device, ++ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ } ++ else ++ { ++ fpi_device_verify_report (device, FPI_MATCH_FAIL, matched, NULL); ++ fpi_device_verify_complete (device, NULL); ++ } ++ } ++ else ++ { ++ GPtrArray *gallery = NULL; ++ FpPrint *hit = NULL; ++ ++ fpi_device_get_identify_data (device, &gallery); ++ ++ if (matched != NULL && gallery != NULL) ++ { ++ guint i; ++ ++ for (i = 0; i < gallery->len; i++) ++ { ++ FpPrint *candidate = g_ptr_array_index (gallery, i); ++ ++ if (aurora_print_has_uuid (candidate, res.uuid)) ++ { ++ hit = candidate; ++ break; ++ } ++ } ++ } ++ ++ if (matched == NULL && res.result == AURORA_BIO_NOT_COMPARED) ++ { ++ fpi_device_identify_complete (device, ++ fpi_device_error_new (FP_DEVICE_ERROR_GENERAL)); ++ } ++ else ++ { ++ fpi_device_identify_report (device, hit, matched, NULL); ++ fpi_device_identify_complete (device, NULL); ++ } ++ } ++ ++ self->nonce_valid = FALSE; ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ return G_SOURCE_REMOVE; ++} ++ ++static void ++aurora_match_start (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_verify_start start = { 0 }; ++ FpiDeviceAction action = fpi_device_get_current_action (device); ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_get_verify_data (device, &print); ++ ++ /* An identify against an EMPTY gallery is "no match" by definition, and ++ * answering it needs no hardware at all. ++ * ++ * This is not an optimisation, it is what unblocks enrolment. fprintd runs a ++ * duplicate check before each enrol by identifying the incoming finger ++ * against the prints already stored. On a device with nothing enrolled that ++ * gallery is empty, so the question has only one possible answer -- but we ++ * were forwarding it to the kernel, whose verify ioctl returns ENOSYS ++ * because matching is not implemented yet. fprintd then reported ++ * "ioctl failed: Function not implemented", which GNOME shows to the user as ++ * "Fingerprint device disconnected". ++ * ++ * So the first enrolment on a fresh device was impossible, and the message ++ * blamed the cable. Answer the empty case here and the enrol proceeds. ++ * ++ * A NON-empty gallery still goes to the kernel and still fails until match ++ * is implemented. That is correct: we must never report "no match" for a ++ * gallery we did not actually search. Reporting a *negative* we can prove is ++ * safe; inventing one we cannot is how a biometric stack silently stops ++ * authenticating anyone. ++ */ ++ if (action == FPI_DEVICE_ACTION_IDENTIFY) ++ { ++ GPtrArray *gallery = NULL; ++ ++ fpi_device_get_identify_data (device, &gallery); ++ if (gallery == NULL || gallery->len == 0) ++ { ++ fp_dbg ("identify against an empty gallery: reporting no match " ++ "without touching the sensor"); ++ fpi_device_identify_report (device, NULL, NULL, NULL); ++ fpi_device_identify_complete (device, NULL); ++ return; ++ } ++ } ++ ++ /* A fresh nonce per operation is what makes the result non-replayable. If we ++ * cannot get one we must not fall back to anything weaker. */ ++ if (getrandom (self->nonce, sizeof (self->nonce), 0) != (gssize) sizeof (self->nonce)) ++ { ++ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_GENERAL, ++ "cannot obtain a nonce for the match"); ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ self->nonce_valid = TRUE; ++ ++ memcpy (start.nonce, self->nonce, sizeof (start.nonce)); ++ ++ if (!aurora_ioctl (self, AURORA_BIO_VERIFY_START, &start, &error)) ++ { ++ self->nonce_valid = FALSE; ++ if (action == FPI_DEVICE_ACTION_VERIFY) ++ fpi_device_verify_complete (device, g_steal_pointer (&error)); ++ else ++ fpi_device_identify_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ aurora_stop_watch (self); ++ self->watch_id = g_unix_fd_add (self->fd, G_IO_IN, aurora_verify_ready, device); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* storage */ ++/* ------------------------------------------------------------------ */ ++ ++static void ++aurora_list (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_list list = { 0 }; ++ g_autoptr(GError) error = NULL; ++ GPtrArray *prints; ++ guint32 i; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_LIST, &list, &error)) ++ { ++ fpi_device_list_complete (device, NULL, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (list.count > AURORA_BIO_MAX_IDENTITIES) ++ { ++ fpi_device_list_complete (device, NULL, ++ fpi_device_error_new_msg (FP_DEVICE_ERROR_PROTO, ++ "device reported %u identities, " ++ "more than the interface allows", ++ list.count)); ++ return; ++ } ++ ++ prints = g_ptr_array_new_with_free_func (g_object_unref); ++ ++ for (i = 0; i < list.count; i++) ++ g_ptr_array_add (prints, ++ g_object_ref_sink (aurora_print_from_uuid (self, ++ list.id[i].uuid, ++ list.id[i].label))); ++ ++ fpi_device_list_complete (device, prints, NULL); ++} ++ ++static void ++aurora_delete (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ struct aurora_bio_delete del = { 0 }; ++ g_autoptr(GError) error = NULL; ++ FpPrint *print = NULL; ++ ++ fpi_device_get_delete_data (device, &print); ++ ++ if (print == NULL || !aurora_uuid_from_print (print, del.uuid, &error)) ++ { ++ if (error == NULL) ++ error = fpi_device_error_new_msg (FP_DEVICE_ERROR_DATA_INVALID, ++ "no print to delete"); ++ fpi_device_delete_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ if (!aurora_ioctl (self, AURORA_BIO_DELETE, &del, &error)) ++ { ++ fpi_device_delete_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ fpi_device_delete_complete (device, NULL); ++} ++ ++static void ++aurora_clear_storage (FpDevice *device) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (device); ++ g_autoptr(GError) error = NULL; ++ ++ if (!aurora_ioctl (self, AURORA_BIO_DELETE_ALL, NULL, &error)) ++ { ++ fpi_device_clear_storage_complete (device, g_steal_pointer (&error)); ++ return; ++ } ++ ++ fpi_device_clear_storage_complete (device, NULL); ++} ++ ++/* ------------------------------------------------------------------ */ ++/* class */ ++/* ------------------------------------------------------------------ */ ++ ++static const FpIdEntry aurora_id_table[] = { ++ { .udev_types = FPI_DEVICE_UDEV_SUBTYPE_MISC, .misc_name = "sep-bio" }, ++ { .udev_types = 0 } ++}; ++ ++static void ++fpi_device_aurora_init (FpiDeviceAurora *self) ++{ ++ self->fd = -1; ++ self->watch_id = 0; ++ self->nonce_valid = FALSE; ++} ++ ++static void ++fpi_device_aurora_finalize (GObject *object) ++{ ++ FpiDeviceAurora *self = FPI_DEVICE_AURORA (object); ++ ++ aurora_stop_watch (self); ++ ++ if (self->fd >= 0) ++ { ++ close (self->fd); ++ self->fd = -1; ++ } ++ ++ memset (self->nonce, 0, sizeof (self->nonce)); ++ ++ G_OBJECT_CLASS (fpi_device_aurora_parent_class)->finalize (object); ++} ++ ++static void ++fpi_device_aurora_class_init (FpiDeviceAuroraClass *klass) ++{ ++ GObjectClass *object_class = G_OBJECT_CLASS (klass); ++ FpDeviceClass *dev_class = FP_DEVICE_CLASS (klass); ++ ++ object_class->finalize = fpi_device_aurora_finalize; ++ ++ dev_class->id = FP_COMPONENT; ++ dev_class->full_name = "Apple secure enclave fingerprint sensor"; ++ dev_class->type = FP_DEVICE_TYPE_UDEV; ++ dev_class->id_table = aurora_id_table; ++ dev_class->scan_type = FP_SCAN_TYPE_PRESS; ++ dev_class->nr_enroll_stages = 8; ++ dev_class->temp_hot_seconds = -1; ++ ++ dev_class->probe = aurora_probe; ++ dev_class->open = aurora_open; ++ dev_class->close = aurora_close; ++ dev_class->cancel = aurora_cancel; ++ dev_class->enroll = aurora_enroll; ++ dev_class->verify = aurora_match_start; ++ dev_class->identify = aurora_match_start; ++ dev_class->list = aurora_list; ++ dev_class->delete = aurora_delete; ++ dev_class->clear_storage = aurora_clear_storage; ++ ++ fpi_device_class_auto_initialize_features (dev_class); ++ ++ /* DUPLICATES_CHECK is deliberately NOT advertised. ++ * ++ * It tells libfprint the device can say whether a finger being enrolled is ++ * already enrolled, and libfprint implements that by running a full IDENTIFY ++ * pass immediately before every enrol. Matching is implemented, but the ++ * enclave's enrol result does not identify an already-enrolled finger in a ++ * way that satisfies libfprint's pre-enrol duplicate-check contract. Keep ++ * the feature unset until that narrower contract is implemented. ++ */ ++} +diff --git a/libfprint/fp-context.c b/libfprint/fp-context.c +index 70d4062..6fdf53b 100644 +--- a/libfprint/fp-context.c ++++ b/libfprint/fp-context.c +@@ -479,6 +479,7 @@ fp_context_enumerate (FpContext *context) + + g_autoptr(GList) spidev_devices = g_udev_client_query_by_subsystem (udev_client, "spidev"); + g_autoptr(GList) hidraw_devices = g_udev_client_query_by_subsystem (udev_client, "hidraw"); ++ g_autoptr(GList) misc_devices = g_udev_client_query_by_subsystem (udev_client, "misc"); + + /* for each potential driver, try to match all requested resources. */ + for (i = 0; i < priv->drivers->len; i++) +@@ -492,7 +493,7 @@ fp_context_enumerate (FpContext *context) + + for (entry = cls->id_table; entry->udev_types; entry++) + { +- GList *matched_spidev = NULL, *matched_hidraw = NULL; ++ GList *matched_spidev = NULL, *matched_hidraw = NULL, *matched_misc = NULL; + + if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_SPIDEV) + { +@@ -530,6 +531,20 @@ fp_context_enumerate (FpContext *context) + if (matched_hidraw == NULL) + continue; + } ++ if (entry->udev_types & FPI_DEVICE_UDEV_SUBTYPE_MISC) ++ { ++ for (matched_misc = misc_devices; matched_misc; matched_misc = matched_misc->next) ++ { ++ const gchar * name = g_udev_device_get_name (matched_misc->data); ++ if (!name || !entry->misc_name) ++ continue; ++ if (g_strcmp0 (name, entry->misc_name) == 0) ++ break; ++ } ++ /* If match was not found exit */ ++ if (matched_misc == NULL) ++ continue; ++ } + priv->pending_devices++; + g_async_initable_new_async (driver, + G_PRIORITY_LOW, +@@ -539,6 +554,7 @@ fp_context_enumerate (FpContext *context) + "fpi-driver-data", entry->driver_data, + "fpi-udev-data-spidev", (matched_spidev ? g_udev_device_get_device_file (matched_spidev->data) : NULL), + "fpi-udev-data-hidraw", (matched_hidraw ? g_udev_device_get_device_file (matched_hidraw->data) : NULL), ++ "fpi-udev-data-misc", (matched_misc ? g_udev_device_get_device_file (matched_misc->data) : NULL), + NULL); + /* remove entries from list to avoid conflicts */ + if (matched_spidev) +@@ -551,6 +567,11 @@ fp_context_enumerate (FpContext *context) + g_object_unref (matched_hidraw->data); + hidraw_devices = g_list_delete_link (hidraw_devices, matched_hidraw); + } ++ if (matched_misc) ++ { ++ g_object_unref (matched_misc->data); ++ misc_devices = g_list_delete_link (misc_devices, matched_misc); ++ } + } + } + +diff --git a/libfprint/fp-device-private.h b/libfprint/fp-device-private.h +index 1c3702f..c8253c6 100644 +--- a/libfprint/fp-device-private.h ++++ b/libfprint/fp-device-private.h +@@ -49,6 +49,7 @@ typedef struct + { + gchar *spidev_path; + gchar *hidraw_path; ++ gchar *misc_path; + } udev_data; + + gboolean is_removed; +diff --git a/libfprint/fp-device.c b/libfprint/fp-device.c +index 115063d..70b10f1 100644 +--- a/libfprint/fp-device.c ++++ b/libfprint/fp-device.c +@@ -53,6 +53,7 @@ enum { + PROP_FPI_USB_DEVICE, + PROP_FPI_UDEV_DATA_SPIDEV, + PROP_FPI_UDEV_DATA_HIDRAW, ++ PROP_FPI_UDEV_DATA_MISC, + PROP_FPI_DRIVER_DATA, + N_PROPS + }; +@@ -237,6 +238,7 @@ fp_device_finalize (GObject *object) + g_clear_pointer (&priv->virtual_env, g_free); + g_clear_pointer (&priv->udev_data.spidev_path, g_free); + g_clear_pointer (&priv->udev_data.hidraw_path, g_free); ++ g_clear_pointer (&priv->udev_data.misc_path, g_free); + + G_OBJECT_CLASS (fp_device_parent_class)->finalize (object); + } +@@ -307,6 +309,13 @@ fp_device_get_property (GObject *object, + g_value_set_string (value, NULL); + break; + ++ case PROP_FPI_UDEV_DATA_MISC: ++ if (cls->type == FP_DEVICE_TYPE_UDEV) ++ g_value_set_string (value, priv->udev_data.misc_path); ++ else ++ g_value_set_string (value, NULL); ++ break; ++ + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + } +@@ -353,6 +362,13 @@ fp_device_set_property (GObject *object, + g_assert (g_value_get_string (value) == NULL); + break; + ++ case PROP_FPI_UDEV_DATA_MISC: ++ if (cls->type == FP_DEVICE_TYPE_UDEV) ++ priv->udev_data.misc_path = g_value_dup_string (value); ++ else ++ g_assert (g_value_get_string (value) == NULL); ++ break; ++ + case PROP_FPI_DRIVER_DATA: + priv->driver_data = g_value_get_uint64 (value); + break; +@@ -583,6 +599,19 @@ fp_device_class_init (FpDeviceClass *klass) + "Private: The path to /dev/hidrawN", + NULL, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); ++ /** ++ * FpDevice::fpi-udev-data-misc: (skip) ++ * ++ * This property is only for internal purposes. ++ * ++ * Stability: private ++ */ ++ properties[PROP_FPI_UDEV_DATA_MISC] = ++ g_param_spec_string ("fpi-udev-data-misc", ++ "Udev data: misc path", ++ "Private: The path to the misc character device", ++ NULL, ++ G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY); + + /** + * FpDevice::fpi-driver-data: (skip) +diff --git a/libfprint/fpi-device.c b/libfprint/fpi-device.c +index 9500b3a..a70db5c 100644 +--- a/libfprint/fpi-device.c ++++ b/libfprint/fpi-device.c +@@ -521,6 +521,9 @@ fpi_device_get_udev_data (FpDevice *device, FpiDeviceUdevSubtypeFlags subtype) + case FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: + return priv->udev_data.spidev_path; + ++ case FPI_DEVICE_UDEV_SUBTYPE_MISC: ++ return priv->udev_data.misc_path; ++ + default: + g_return_val_if_reached (NULL); + return NULL; +diff --git a/libfprint/fpi-device.h b/libfprint/fpi-device.h +index b17c10d..7671ad6 100644 +--- a/libfprint/fpi-device.h ++++ b/libfprint/fpi-device.h +@@ -28,12 +28,14 @@ + * FpiDeviceUdevSubtypeFlags: + * @FPI_DEVICE_UDEV_SUBTYPE_SPIDEV: The device requires an spidev node + * @FPI_DEVICE_UDEV_SUBTYPE_HIDRAW: The device requires a hidraw node ++ * @FPI_DEVICE_UDEV_SUBTYPE_MISC: The device requires a misc character node + * + * Bitfield of required hardware resources for a udev-backed device. + */ + typedef enum { + FPI_DEVICE_UDEV_SUBTYPE_SPIDEV = 1 << 0, + FPI_DEVICE_UDEV_SUBTYPE_HIDRAW = 1 << 1, ++ FPI_DEVICE_UDEV_SUBTYPE_MISC = 1 << 2, + } FpiDeviceUdevSubtypeFlags; + + /** +@@ -71,6 +73,7 @@ struct _FpIdEntry + guint pid; + guint vid; + } hid_id; ++ const gchar *misc_name; + }; + }; + guint64 driver_data; +diff --git a/libfprint/meson.build b/libfprint/meson.build +index f11533c..400c5b7 100644 +--- a/libfprint/meson.build ++++ b/libfprint/meson.build +@@ -116,6 +116,7 @@ driver_sources = { + ), + 'etes603' : files('drivers/etes603.c'), + 'egis0570' : files('drivers/egis0570.c'), ++ 'aurora' : files('drivers/aurora/aurora.c'), + 'egismoc' : files('drivers/egismoc/egismoc.c'), + 'egis_etu905' : files('drivers/egismoc/egis_etu905.c'), + 'vfs0050' : files('drivers/vfs0050.c'), +diff --git a/meson.build b/meson.build +index ab09dc1..dfa7460 100644 +--- a/meson.build ++++ b/meson.build +@@ -151,6 +151,9 @@ drivers_info = { + # SPI driver (non-optional if SPI is available) + 'elanspi': { 'spi': true, 'helper': ['udev'], 'optional': not have_spi }, + ++ # Apple secure enclave, reached through a kernel misc device ++ 'aurora': { 'helper': ['udev'], 'optional': host_machine.system() != 'linux' }, ++ + # Virtual drivers (test-only, optional) + 'virtual_image': { 'virtual': true, 'helper': ['virtual'], 'optional': true }, + 'virtual_device': { 'virtual': true, 'helper': ['virtual'], 'optional': true },